Merge branch 'dm-application-worker' into 'master'

Add ApplicationWorker and make every worker include it

See merge request gitlab-org/gitlab-ce!15632

3 files changed, 76 insertions, 4 deletions

diff --git a/lib/after_commit_queue.rb b/lib/after_commit_queue.rb
index 4750a2c373a..db63c5038ae 100644
--- a/lib/after_commit_queue.rb
+++ b/lib/after_commit_queue.rb
@@ -6,12 +6,34 @@ module AfterCommitQueue
     after_rollback :_clear_after_commit_queue
   end
 
-  def run_after_commit(method = nil, &block)
-    _after_commit_queue << proc { self.send(method) } if method # rubocop:disable GitlabSecurity/PublicSend
+  def run_after_commit(&block)
     _after_commit_queue << block if block
+
+    true
+  end
+
+  def run_after_commit_or_now(&block)
+    if AfterCommitQueue.inside_transaction?
+      run_after_commit(&block)
+    else
+      instance_eval(&block)
+    end
+
     true
   end
 
+  def self.open_transactions_baseline
+    if ::Rails.env.test?
+      return DatabaseCleaner.connections.count { |conn| conn.strategy.is_a?(DatabaseCleaner::ActiveRecord::Transaction) }
+    end
+
+    0
+  end
+
+  def self.inside_transaction?
+    ActiveRecord::Base.connection.open_transactions > open_transactions_baseline
+  end
+
   protected
 
   def _run_after_commit_queue
diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb
index c276c3566b4..3f65bc912de 100644
--- a/lib/gitlab/database/migration_helpers.rb
+++ b/lib/gitlab/database/migration_helpers.rb
@@ -703,14 +703,14 @@ into similar problems in the future (e.g. when new tables are created).
             # We push multiple jobs at a time to reduce the time spent in
             # Sidekiq/Redis operations. We're using this buffer based approach so we
             # don't need to run additional queries for every range.
-            BackgroundMigrationWorker.perform_bulk(jobs)
+            BackgroundMigrationWorker.bulk_perform_async(jobs)
             jobs.clear
           end
 
           jobs << [job_class_name, [start_id, end_id]]
         end
 
-        BackgroundMigrationWorker.perform_bulk(jobs) unless jobs.empty?
+        BackgroundMigrationWorker.bulk_perform_async(jobs) unless jobs.empty?
       end
 
       # Queues background migration jobs for an entire table, batched by ID range.
diff --git a/lib/gitlab/sidekiq_config.rb b/lib/gitlab/sidekiq_config.rb
new file mode 100644
index 00000000000..dc9886732b5
--- /dev/null
+++ b/lib/gitlab/sidekiq_config.rb
@@ -0,0 +1,50 @@
+require 'yaml'
+
+module Gitlab
+  module SidekiqConfig
+    def self.redis_queues
+      @redis_queues ||= Sidekiq::Queue.all.map(&:name)
+    end
+
+    # This method is called by `bin/sidekiq-cluster` in EE, which runs outside
+    # of bundler/Rails context, so we cannot use any gem or Rails methods.
+    def self.config_queues(rails_path = Rails.root.to_s)
+      @config_queues ||= begin
+        config = YAML.load_file(File.join(rails_path, 'config', 'sidekiq_queues.yml'))
+        config[:queues].map(&:first)
+      end
+    end
+
+    def self.cron_workers
+      @cron_workers ||= Settings.cron_jobs.map { |job_name, options| options['job_class'].constantize }
+    end
+
+    def self.workers
+      @workers ||= find_workers(Rails.root.join('app', 'workers'))
+    end
+
+    def self.default_queues
+      [ActionMailer::DeliveryJob.queue_name, 'default']
+    end
+
+    def self.worker_queues
+      @worker_queues ||= (workers.map(&:queue) + default_queues).uniq
+    end
+
+    def self.find_workers(root)
+      concerns = root.join('concerns').to_s
+
+      workers = Dir[root.join('**', '*.rb')]
+        .reject { |path| path.start_with?(concerns) }
+
+      workers.map! do |path|
+        ns = Pathname.new(path).relative_path_from(root).to_s.gsub('.rb', '')
+
+        ns.camelize.constantize
+      end
+
+      # Skip concerns
+      workers.select { |w| w < Sidekiq::Worker }
+    end
+  end
+end