Add latest changes from gitlab-org/gitlab@master

5 files changed, 145 insertions, 12 deletions

diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml
index 2acb79e3e22..afa575241a1 100644
--- a/lib/gitlab/import_export/import_export.yml
+++ b/lib/gitlab/import_export/import_export.yml
@@ -178,7 +178,6 @@ excluded_attributes:
     - :encrypted_secret_token
     - :encrypted_secret_token_iv
     - :repository_storage
-    - :storage_version
   merge_request_diff:
     - :external_diff
     - :stored_externally
diff --git a/lib/gitlab/sidekiq_config.rb b/lib/gitlab/sidekiq_config.rb
index b246c507e9e..c96212f27a7 100644
--- a/lib/gitlab/sidekiq_config.rb
+++ b/lib/gitlab/sidekiq_config.rb
@@ -4,6 +4,22 @@ require 'yaml'
 
 module Gitlab
   module SidekiqConfig
+    FOSS_QUEUE_CONFIG_PATH = 'app/workers/all_queues.yml'
+    EE_QUEUE_CONFIG_PATH = 'ee/app/workers/all_queues.yml'
+
+    QUEUE_CONFIG_PATHS = [
+      FOSS_QUEUE_CONFIG_PATH,
+      (EE_QUEUE_CONFIG_PATH if Gitlab.ee?)
+    ].compact.freeze
+
+    # For queues that don't have explicit workers - default and mailers
+    DummyWorker = Struct.new(:queue)
+
+    DEFAULT_WORKERS = [
+      Gitlab::SidekiqConfig::Worker.new(DummyWorker.new('default'), ee: false),
+      Gitlab::SidekiqConfig::Worker.new(DummyWorker.new('mailers'), ee: false)
+    ].freeze
+
     class << self
       include Gitlab::SidekiqConfig::CliMethods
 
@@ -25,28 +41,46 @@ module Gitlab
 
       def workers
         @workers ||= begin
-          result = find_workers(Rails.root.join('app', 'workers'))
-          result.concat(find_workers(Rails.root.join('ee', 'app', 'workers'))) if Gitlab.ee?
+          result = []
+          result.concat(DEFAULT_WORKERS)
+          result.concat(find_workers(Rails.root.join('app', 'workers'), ee: false))
+
+          if Gitlab.ee?
+            result.concat(find_workers(Rails.root.join('ee', 'app', 'workers'), ee: true))
+          end
+
           result
         end
       end
 
+      def workers_for_all_queues_yml
+        workers.partition(&:ee?).reverse.map(&:sort)
+      end
+
+      def all_queues_yml_outdated?
+        foss_workers, ee_workers = workers_for_all_queues_yml
+
+        return true if foss_workers != YAML.safe_load(File.read(FOSS_QUEUE_CONFIG_PATH))
+
+        Gitlab.ee? && ee_workers != YAML.safe_load(File.read(EE_QUEUE_CONFIG_PATH))
+      end
+
       private
 
-      def find_workers(root)
+      def find_workers(root, ee:)
         concerns = root.join('concerns').to_s
 
-        workers = Dir[root.join('**', '*.rb')]
+        Dir[root.join('**', '*.rb')]
           .reject { |path| path.start_with?(concerns) }
+          .map { |path| worker_from_path(path, root) }
+          .select { |worker| worker < Sidekiq::Worker }
+          .map { |worker| Gitlab::SidekiqConfig::Worker.new(worker, ee: ee) }
+      end
 
-        workers.map! do |path|
-          ns = Pathname.new(path).relative_path_from(root).to_s.gsub('.rb', '')
-
-          ns.camelize.constantize
-        end
+      def worker_from_path(path, root)
+        ns = Pathname.new(path).relative_path_from(root).to_s.gsub('.rb', '')
 
-        # Skip things that aren't workers
-        workers.select { |w| w < Sidekiq::Worker }
+        ns.camelize.constantize
       end
     end
   end
diff --git a/lib/gitlab/sidekiq_config/worker.rb b/lib/gitlab/sidekiq_config/worker.rb
new file mode 100644
index 00000000000..313d9b17b16
--- /dev/null
+++ b/lib/gitlab/sidekiq_config/worker.rb
@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module SidekiqConfig
+    class Worker
+      include Comparable
+
+      attr_reader :klass
+      delegate :feature_category_not_owned?, :get_feature_category,
+               :get_worker_resource_boundary, :latency_sensitive_worker?,
+               :queue, :worker_has_external_dependencies?,
+               to: :klass
+
+      def initialize(klass, ee:)
+        @klass = klass
+        @ee = ee
+      end
+
+      def ee?
+        @ee
+      end
+
+      def ==(other)
+        to_yaml == case other
+                   when self.class
+                     other.to_yaml
+                   else
+                     other
+                   end
+      end
+
+      def <=>(other)
+        to_sort <=> other.to_sort
+      end
+
+      # Put namespaced queues first
+      def to_sort
+        [queue.include?(':') ? 0 : 1, queue]
+      end
+
+      # YAML representation
+      def encode_with(coder)
+        coder.represent_scalar(nil, to_yaml)
+      end
+
+      def to_yaml
+        queue
+      end
+    end
+  end
+end
diff --git a/lib/tasks/gitlab/sidekiq.rake b/lib/tasks/gitlab/sidekiq.rake
new file mode 100644
index 00000000000..f6bb0196236
--- /dev/null
+++ b/lib/tasks/gitlab/sidekiq.rake
@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+return if Rails.env.production?
+
+namespace :gitlab do
+  namespace :sidekiq do
+    namespace :all_queues_yml do
+      def write_yaml(path, object)
+        banner = <<~BANNER
+          # This file is generated automatically by
+          #   bin/rake gitlab:sidekiq:all_queues_yml:generate
+          #
+          # Do not edit it manually!
+        BANNER
+
+        File.write(path, banner + YAML.dump(object))
+      end
+
+      desc 'GitLab | Generate all_queues.yml based on worker definitions'
+      task generate: :environment do
+        foss_workers, ee_workers = Gitlab::SidekiqConfig.workers_for_all_queues_yml
+
+        write_yaml(Gitlab::SidekiqConfig::FOSS_QUEUE_CONFIG_PATH, foss_workers)
+
+        if Gitlab.ee?
+          write_yaml(Gitlab::SidekiqConfig::EE_QUEUE_CONFIG_PATH, ee_workers)
+        end
+      end
+
+      desc 'GitLab | Validate that all_queues.yml matches worker definitions'
+      task check: :environment do
+        if Gitlab::SidekiqConfig.all_queues_yml_outdated?
+          raise <<~MSG
+            Changes in worker queues found, please update the metadata by running:
+
+              bin/rake gitlab:sidekiq:all_queues_yml:generate
+
+            Then commit and push the changes from:
+
+            - #{Gitlab::SidekiqConfig::FOSS_QUEUE_CONFIG_PATH}
+            - #{Gitlab::SidekiqConfig::EE_QUEUE_CONFIG_PATH}
+
+          MSG
+        end
+      end
+    end
+  end
+end
diff --git a/lib/tasks/lint.rake b/lib/tasks/lint.rake
index 9a5693e78a2..42a9c027b6a 100644
--- a/lib/tasks/lint.rake
+++ b/lib/tasks/lint.rake
@@ -34,6 +34,7 @@ unless Rails.env.production?
         scss_lint
         gettext:lint
         lint:static_verification
+        gitlab:sidekiq:all_queues_yml:check
       ]
 
       if Gitlab.ee?