diff options
-rw-r--r-- | config/initializers/1_settings.rb | 2 | ||||
-rw-r--r-- | config/settings.rb | 6 | ||||
-rw-r--r-- | spec/config/settings_spec.rb | 14 | ||||
-rw-r--r-- | spec/lib/gitlab/sidekiq_middleware/duplicate_jobs/server_spec.rb | 8 | ||||
-rw-r--r-- | spec/lib/gitlab/sidekiq_migrate_jobs_spec.rb | 8 | ||||
-rw-r--r-- | spec/lib/gitlab/sidekiq_queue_spec.rb | 10 | ||||
-rw-r--r-- | spec/support/rspec_order_todo.yml | 5 | ||||
-rw-r--r-- | spec/workers/concerns/cluster_agent_queue_spec.rb | 1 | ||||
-rw-r--r-- | spec/workers/concerns/cluster_queue_spec.rb | 21 | ||||
-rw-r--r-- | spec/workers/concerns/cronjob_queue_spec.rb | 4 | ||||
-rw-r--r-- | spec/workers/concerns/gitlab/github_import/queue_spec.rb | 18 | ||||
-rw-r--r-- | spec/workers/concerns/pipeline_background_queue_spec.rb | 21 | ||||
-rw-r--r-- | spec/workers/concerns/pipeline_queue_spec.rb | 21 | ||||
-rw-r--r-- | spec/workers/concerns/repository_check_queue_spec.rb | 4 | ||||
-rw-r--r-- | spec/workers/concerns/waitable_worker_spec.rb | 3 | ||||
-rw-r--r-- | spec/workers/ssh_keys/expired_notification_worker_spec.rb | 1 | ||||
-rw-r--r-- | spec/workers/ssh_keys/expiring_soon_notification_worker_spec.rb | 1 |
17 files changed, 113 insertions, 35 deletions
diff --git a/config/initializers/1_settings.rb b/config/initializers/1_settings.rb index 45307c3454b..2244e415c3d 100644 --- a/config/initializers/1_settings.rb +++ b/config/initializers/1_settings.rb @@ -814,7 +814,7 @@ end # Settings['sidekiq'] ||= Settingslogic.new({}) Settings['sidekiq']['log_format'] ||= 'default' -Settings['sidekiq']['routing_rules'] = Settings.__send__(:build_sidekiq_routing_rules, Settings['sidekiq']['routing_rules']) +Settings['sidekiq']['routing_rules'] ||= [] # # GitLab Shell diff --git a/config/settings.rb b/config/settings.rb index b242e970cc6..51d54817646 100644 --- a/config/settings.rb +++ b/config/settings.rb @@ -204,11 +204,5 @@ class Settings < Settingslogic "#{minute} #{hour} * * #{day_of_week}" end - - # Route all jobs to 'default' queue. This setting is meant for self-managed instances use to keep things simple. - # See https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/1491 - def build_sidekiq_routing_rules(rules) - rules.nil? || rules&.empty? ? [['*', 'default']] : rules - end end end diff --git a/spec/config/settings_spec.rb b/spec/config/settings_spec.rb index 9b721d8cfca..fe2081fa5de 100644 --- a/spec/config/settings_spec.rb +++ b/spec/config/settings_spec.rb @@ -151,20 +151,6 @@ RSpec.describe Settings do end end - describe '.build_sidekiq_routing_rules' do - [ - [nil, [['*', 'default']]], - [[], [['*', 'default']]], - [[['name=foobar', 'foobar']], [['name=foobar', 'foobar']]] - ].each do |input_rules, output_rules| - context "Given input routing_rules #{input_rules}" do - it "returns output routing_rules #{output_rules}" do - expect(described_class.send(:build_sidekiq_routing_rules, input_rules)).to eq(output_rules) - end - end - end - end - describe '.microsoft_graph_mailer' do it 'defaults' do expect(described_class.microsoft_graph_mailer.enabled).to be false diff --git a/spec/lib/gitlab/sidekiq_middleware/duplicate_jobs/server_spec.rb b/spec/lib/gitlab/sidekiq_middleware/duplicate_jobs/server_spec.rb index cc730e203f6..09548d21106 100644 --- a/spec/lib/gitlab/sidekiq_middleware/duplicate_jobs/server_spec.rb +++ b/spec/lib/gitlab/sidekiq_middleware/duplicate_jobs/server_spec.rb @@ -41,10 +41,10 @@ RSpec.describe Gitlab::SidekiqMiddleware::DuplicateJobs::Server, :clean_gitlab_r describe '#call' do it 'removes the stored job from redis before execution' do bare_job = { 'class' => 'TestDeduplicationWorker', 'args' => ['hello'] } - job_definition = Gitlab::SidekiqMiddleware::DuplicateJobs::DuplicateJob.new(bare_job.dup, 'default') + job_definition = Gitlab::SidekiqMiddleware::DuplicateJobs::DuplicateJob.new(bare_job.dup, 'test_deduplication') expect(Gitlab::SidekiqMiddleware::DuplicateJobs::DuplicateJob) - .to receive(:new).with(a_hash_including(bare_job), 'default') + .to receive(:new).with(a_hash_including(bare_job), 'test_deduplication') .and_return(job_definition).twice # once in client middleware expect(job_definition).to receive(:delete!).ordered.and_call_original @@ -60,10 +60,10 @@ RSpec.describe Gitlab::SidekiqMiddleware::DuplicateJobs::Server, :clean_gitlab_r it 'removes the stored job from redis after execution' do bare_job = { 'class' => 'TestDeduplicationWorker', 'args' => ['hello'] } - job_definition = Gitlab::SidekiqMiddleware::DuplicateJobs::DuplicateJob.new(bare_job.dup, 'default') + job_definition = Gitlab::SidekiqMiddleware::DuplicateJobs::DuplicateJob.new(bare_job.dup, 'test_deduplication') expect(Gitlab::SidekiqMiddleware::DuplicateJobs::DuplicateJob) - .to receive(:new).with(a_hash_including(bare_job), 'default') + .to receive(:new).with(a_hash_including(bare_job), 'test_deduplication') .and_return(job_definition).twice # once in client middleware expect(TestDeduplicationWorker).to receive(:work).ordered.and_call_original diff --git a/spec/lib/gitlab/sidekiq_migrate_jobs_spec.rb b/spec/lib/gitlab/sidekiq_migrate_jobs_spec.rb index a576cf3e2ab..d4391d3023a 100644 --- a/spec/lib/gitlab/sidekiq_migrate_jobs_spec.rb +++ b/spec/lib/gitlab/sidekiq_migrate_jobs_spec.rb @@ -46,7 +46,7 @@ RSpec.describe Gitlab::SidekiqMigrateJobs, :clean_gitlab_redis_queues do expect(migrator.execute('PostReceive' => 'new_queue')).to eq(scanned: 3, migrated: 0) expect(set_after.length).to eq(3) - expect(set_after.map(&:first)).to all(include('queue' => 'default', + expect(set_after.map(&:first)).to all(include('queue' => 'authorized_projects', 'class' => 'AuthorizedProjectsWorker')) end end @@ -62,7 +62,7 @@ RSpec.describe Gitlab::SidekiqMigrateJobs, :clean_gitlab_redis_queues do if item['class'] == 'AuthorizedProjectsWorker' expect(item).to include('queue' => 'new_queue', 'args' => [i]) else - expect(item).to include('queue' => 'default', 'args' => [i]) + expect(item).to include('queue' => 'post_receive', 'args' => [i]) end expect(score).to be_within(schedule_jitter).of(i.succ.hours.from_now.to_i) @@ -116,7 +116,7 @@ RSpec.describe Gitlab::SidekiqMigrateJobs, :clean_gitlab_redis_queues do expect(migrator.execute('PostReceive' => 'new_queue')).to eq(scanned: 4, migrated: 0) expect(set_after.length).to eq(3) - expect(set_after.map(&:first)).to all(include('queue' => 'default')) + expect(set_after.map(&:first)).to all(include('queue' => 'authorized_projects')) end end @@ -138,7 +138,7 @@ RSpec.describe Gitlab::SidekiqMigrateJobs, :clean_gitlab_redis_queues do expect(migrator.execute('PostReceive' => 'new_queue')).to eq(scanned: 4, migrated: 1) expect(set_after.group_by { |job| job.first['queue'] }.transform_values(&:count)) - .to eq('default' => 6, 'new_queue' => 1) + .to eq('authorized_projects' => 6, 'new_queue' => 1) end it 'iterates through the entire set of jobs' do diff --git a/spec/lib/gitlab/sidekiq_queue_spec.rb b/spec/lib/gitlab/sidekiq_queue_spec.rb index 93632848788..5e91282612e 100644 --- a/spec/lib/gitlab/sidekiq_queue_spec.rb +++ b/spec/lib/gitlab/sidekiq_queue_spec.rb @@ -4,15 +4,15 @@ require 'spec_helper' RSpec.describe Gitlab::SidekiqQueue, :clean_gitlab_redis_queues do around do |example| - Sidekiq::Queue.new('foobar').clear + Sidekiq::Queue.new('default').clear Sidekiq::Testing.disable!(&example) - Sidekiq::Queue.new('foobar').clear + Sidekiq::Queue.new('default').clear end def add_job(args, user:, klass: 'AuthorizedProjectsWorker') Sidekiq::Client.push( 'class' => klass, - 'queue' => 'foobar', + 'queue' => 'default', 'args' => args, 'meta.user' => user.username ) @@ -20,7 +20,7 @@ RSpec.describe Gitlab::SidekiqQueue, :clean_gitlab_redis_queues do describe '#drop_jobs!' do shared_examples 'queue processing' do - let(:sidekiq_queue) { described_class.new('foobar') } + let(:sidekiq_queue) { described_class.new('default') } let_it_be(:sidekiq_queue_user) { create(:user) } before do @@ -80,7 +80,7 @@ RSpec.describe Gitlab::SidekiqQueue, :clean_gitlab_redis_queues do it 'raises NoMetadataError' do add_job([1], user: create(:user)) - expect { described_class.new('foobar').drop_jobs!({ username: 'sidekiq_queue_user' }, timeout: 1) } + expect { described_class.new('default').drop_jobs!({ username: 'sidekiq_queue_user' }, timeout: 1) } .to raise_error(described_class::NoMetadataError) end end diff --git a/spec/support/rspec_order_todo.yml b/spec/support/rspec_order_todo.yml index c4377e368ee..30220b04fa2 100644 --- a/spec/support/rspec_order_todo.yml +++ b/spec/support/rspec_order_todo.yml @@ -3442,6 +3442,7 @@ - './ee/spec/workers/concerns/elastic/indexing_control_spec.rb' - './ee/spec/workers/concerns/elastic/migration_obsolete_spec.rb' - './ee/spec/workers/concerns/elastic/migration_options_spec.rb' +- './ee/spec/workers/concerns/geo_queue_spec.rb' - './ee/spec/workers/concerns/update_orchestration_policy_configuration_spec.rb' - './ee/spec/workers/create_github_webhook_worker_spec.rb' - './ee/spec/workers/deployments/auto_rollback_worker_spec.rb' @@ -10862,14 +10863,18 @@ - './spec/workers/cluster_wait_for_ingress_ip_address_worker_spec.rb' - './spec/workers/concerns/application_worker_spec.rb' - './spec/workers/concerns/cluster_agent_queue_spec.rb' +- './spec/workers/concerns/cluster_queue_spec.rb' - './spec/workers/concerns/cronjob_queue_spec.rb' - './spec/workers/concerns/gitlab/github_import/object_importer_spec.rb' +- './spec/workers/concerns/gitlab/github_import/queue_spec.rb' - './spec/workers/concerns/gitlab/github_import/rescheduling_methods_spec.rb' - './spec/workers/concerns/gitlab/github_import/stage_methods_spec.rb' - './spec/workers/concerns/gitlab/notify_upon_death_spec.rb' - './spec/workers/concerns/limited_capacity/job_tracker_spec.rb' - './spec/workers/concerns/limited_capacity/worker_spec.rb' - './spec/workers/concerns/packages/cleanup_artifact_worker_spec.rb' +- './spec/workers/concerns/pipeline_background_queue_spec.rb' +- './spec/workers/concerns/pipeline_queue_spec.rb' - './spec/workers/concerns/project_import_options_spec.rb' - './spec/workers/concerns/reenqueuer_spec.rb' - './spec/workers/concerns/repository_check_queue_spec.rb' diff --git a/spec/workers/concerns/cluster_agent_queue_spec.rb b/spec/workers/concerns/cluster_agent_queue_spec.rb index 4f67102a0be..b5189cbd8c8 100644 --- a/spec/workers/concerns/cluster_agent_queue_spec.rb +++ b/spec/workers/concerns/cluster_agent_queue_spec.rb @@ -14,5 +14,6 @@ RSpec.describe ClusterAgentQueue do end end + it { expect(worker.queue).to eq('cluster_agent:example') } it { expect(worker.get_feature_category).to eq(:kubernetes_management) } end diff --git a/spec/workers/concerns/cluster_queue_spec.rb b/spec/workers/concerns/cluster_queue_spec.rb new file mode 100644 index 00000000000..c03ca9cea48 --- /dev/null +++ b/spec/workers/concerns/cluster_queue_spec.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe ClusterQueue do + let(:worker) do + Class.new do + def self.name + 'DummyWorker' + end + + include ApplicationWorker + include ClusterQueue + end + end + + it 'sets a default pipelines queue automatically' do + expect(worker.sidekiq_options['queue']) + .to eq 'gcp_cluster:dummy' + end +end diff --git a/spec/workers/concerns/cronjob_queue_spec.rb b/spec/workers/concerns/cronjob_queue_spec.rb index 7dd016fc78a..0244535051f 100644 --- a/spec/workers/concerns/cronjob_queue_spec.rb +++ b/spec/workers/concerns/cronjob_queue_spec.rb @@ -40,6 +40,10 @@ RSpec.describe CronjobQueue do stub_const("AnotherWorker", another_worker) end + it 'sets the queue name of a worker' do + expect(worker.sidekiq_options['queue'].to_s).to eq('cronjob:dummy') + end + it 'disables retrying of failed jobs' do expect(worker.sidekiq_options['retry']).to eq(false) end diff --git a/spec/workers/concerns/gitlab/github_import/queue_spec.rb b/spec/workers/concerns/gitlab/github_import/queue_spec.rb new file mode 100644 index 00000000000..beca221b593 --- /dev/null +++ b/spec/workers/concerns/gitlab/github_import/queue_spec.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::GithubImport::Queue do + it 'sets the Sidekiq options for the worker' do + worker = Class.new do + def self.name + 'DummyWorker' + end + + include ApplicationWorker + include Gitlab::GithubImport::Queue + end + + expect(worker.sidekiq_options['queue']).to eq('github_importer:dummy') + end +end diff --git a/spec/workers/concerns/pipeline_background_queue_spec.rb b/spec/workers/concerns/pipeline_background_queue_spec.rb new file mode 100644 index 00000000000..77c7e7440c5 --- /dev/null +++ b/spec/workers/concerns/pipeline_background_queue_spec.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe PipelineBackgroundQueue do + let(:worker) do + Class.new do + def self.name + 'DummyWorker' + end + + include ApplicationWorker + include PipelineBackgroundQueue + end + end + + it 'sets a default object storage queue automatically' do + expect(worker.sidekiq_options['queue']) + .to eq 'pipeline_background:dummy' + end +end diff --git a/spec/workers/concerns/pipeline_queue_spec.rb b/spec/workers/concerns/pipeline_queue_spec.rb new file mode 100644 index 00000000000..6c1ac2052e4 --- /dev/null +++ b/spec/workers/concerns/pipeline_queue_spec.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe PipelineQueue do + let(:worker) do + Class.new do + def self.name + 'DummyWorker' + end + + include ApplicationWorker + include PipelineQueue + end + end + + it 'sets a default pipelines queue automatically' do + expect(worker.sidekiq_options['queue']) + .to eq 'pipeline_default:dummy' + end +end diff --git a/spec/workers/concerns/repository_check_queue_spec.rb b/spec/workers/concerns/repository_check_queue_spec.rb index 08ac73aac7b..ae377c09b37 100644 --- a/spec/workers/concerns/repository_check_queue_spec.rb +++ b/spec/workers/concerns/repository_check_queue_spec.rb @@ -14,6 +14,10 @@ RSpec.describe RepositoryCheckQueue do end end + it 'sets the queue name of a worker' do + expect(worker.sidekiq_options['queue'].to_s).to eq('repository_check:dummy') + end + it 'disables retrying of failed jobs' do expect(worker.sidekiq_options['retry']).to eq(false) end diff --git a/spec/workers/concerns/waitable_worker_spec.rb b/spec/workers/concerns/waitable_worker_spec.rb index 2df5b60deaf..bf156c3b8cb 100644 --- a/spec/workers/concerns/waitable_worker_spec.rb +++ b/spec/workers/concerns/waitable_worker_spec.rb @@ -49,7 +49,8 @@ RSpec.describe WaitableWorker do expect(Gitlab::AppJsonLogger).to( receive(:info).with(a_hash_including('message' => 'running inline', 'class' => 'Gitlab::Foo::Bar::DummyWorker', - 'job_status' => 'running')) + 'job_status' => 'running', + 'queue' => 'foo_bar_dummy')) .once) worker.bulk_perform_and_wait(args_list) diff --git a/spec/workers/ssh_keys/expired_notification_worker_spec.rb b/spec/workers/ssh_keys/expired_notification_worker_spec.rb index f93d02e86c0..26d9460d73e 100644 --- a/spec/workers/ssh_keys/expired_notification_worker_spec.rb +++ b/spec/workers/ssh_keys/expired_notification_worker_spec.rb @@ -7,6 +7,7 @@ RSpec.describe SshKeys::ExpiredNotificationWorker, type: :worker do it 'uses a cronjob queue' do expect(worker.sidekiq_options_hash).to include( + 'queue' => 'cronjob:ssh_keys_expired_notification', 'queue_namespace' => :cronjob ) end diff --git a/spec/workers/ssh_keys/expiring_soon_notification_worker_spec.rb b/spec/workers/ssh_keys/expiring_soon_notification_worker_spec.rb index ed6701532a5..e907d035020 100644 --- a/spec/workers/ssh_keys/expiring_soon_notification_worker_spec.rb +++ b/spec/workers/ssh_keys/expiring_soon_notification_worker_spec.rb @@ -7,6 +7,7 @@ RSpec.describe SshKeys::ExpiringSoonNotificationWorker, type: :worker do it 'uses a cronjob queue' do expect(worker.sidekiq_options_hash).to include( + 'queue' => 'cronjob:ssh_keys_expiring_soon_notification', 'queue_namespace' => :cronjob ) end |