Use a dedicated queue for each worker

1 files changed, 24 insertions, 13 deletions

diff --git a/lib/gitlab/sidekiq_config.rb b/lib/gitlab/sidekiq_config.rb
index dc9886732b5..c3d7814551c 100644
--- a/lib/gitlab/sidekiq_config.rb
+++ b/lib/gitlab/sidekiq_config.rb
@@ -1,16 +1,35 @@
 require 'yaml'
+require 'set'
 
 module Gitlab
   module SidekiqConfig
-    def self.redis_queues
-      @redis_queues ||= Sidekiq::Queue.all.map(&:name)
+    # This method is called by `bin/sidekiq-cluster` in EE, which runs outside
+    # of bundler/Rails context, so we cannot use any gem or Rails methods.
+    def self.worker_queues(rails_path = Rails.root.to_s)
+      @worker_queues ||= {}
+      @worker_queues[rails_path] ||= YAML.load_file(File.join(rails_path, 'app/workers/all_queues.yml'))
     end
 
     # This method is called by `bin/sidekiq-cluster` in EE, which runs outside
     # of bundler/Rails context, so we cannot use any gem or Rails methods.
-    def self.config_queues(rails_path = Rails.root.to_s)
+    def self.expand_queues(queues, all_queues = self.worker_queues)
+      return [] if queues.empty?
+
+      queues_set = all_queues.to_set
+
+      queues.flat_map do |queue|
+        [queue, *queues_set.grep(/\A#{queue}:/)]
+      end
+    end
+
+    def self.redis_queues
+      # Not memoized, because this can change during the life of the application
+      Sidekiq::Queue.all.map(&:name)
+    end
+
+    def self.config_queues
       @config_queues ||= begin
-        config = YAML.load_file(File.join(rails_path, 'config', 'sidekiq_queues.yml'))
+        config = YAML.load_file(Rails.root.join('config/sidekiq_queues.yml'))
         config[:queues].map(&:first)
       end
     end
@@ -23,14 +42,6 @@ module Gitlab
       @workers ||= find_workers(Rails.root.join('app', 'workers'))
     end
 
-    def self.default_queues
-      [ActionMailer::DeliveryJob.queue_name, 'default']
-    end
-
-    def self.worker_queues
-      @worker_queues ||= (workers.map(&:queue) + default_queues).uniq
-    end
-
     def self.find_workers(root)
       concerns = root.join('concerns').to_s
 
@@ -43,7 +54,7 @@ module Gitlab
         ns.camelize.constantize
       end
 
-      # Skip concerns
+      # Skip things that aren't workers
       workers.select { |w| w < Sidekiq::Worker }
     end
   end