Add latest changes from gitlab-org/gitlab@master

2 files changed, 102 insertions, 74 deletions

diff --git a/lib/gitlab/sidekiq_config/cli_methods.rb b/lib/gitlab/sidekiq_config/cli_methods.rb
index a256632bc12..8eef15f9ccb 100644
--- a/lib/gitlab/sidekiq_config/cli_methods.rb
+++ b/lib/gitlab/sidekiq_config/cli_methods.rb
@@ -12,35 +12,19 @@ module Gitlab
       # rubocop:disable Gitlab/ModuleWithInstanceVariables
       extend self
 
+      # The file names are misleading. Those files contain the metadata of the
+      # workers. They should be renamed to all_workers instead.
+      # https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/1018
       QUEUE_CONFIG_PATHS = begin
         result = %w[app/workers/all_queues.yml]
         result << 'ee/app/workers/all_queues.yml' if Gitlab.ee?
         result
       end.freeze
 
-      QUERY_OR_OPERATOR = '|'
-      QUERY_AND_OPERATOR = '&'
-      QUERY_CONCATENATE_OPERATOR = ','
-      QUERY_TERM_REGEX = %r{^(\w+)(!?=)([\w:#{QUERY_CONCATENATE_OPERATOR}]+)}.freeze
+      def worker_metadatas(rails_path = Rails.root.to_s)
+        @worker_metadatas ||= {}
 
-      QUERY_PREDICATES = {
-        feature_category: :to_sym,
-        has_external_dependencies: lambda { |value| value == 'true' },
-        name: :to_s,
-        resource_boundary: :to_sym,
-        tags: :to_sym,
-        urgency: :to_sym
-      }.freeze
-
-      QueryError = Class.new(StandardError)
-      InvalidTerm = Class.new(QueryError)
-      UnknownOperator = Class.new(QueryError)
-      UnknownPredicate = Class.new(QueryError)
-
-      def all_queues(rails_path = Rails.root.to_s)
-        @worker_queues ||= {}
-
-        @worker_queues[rails_path] ||= QUEUE_CONFIG_PATHS.flat_map do |path|
+        @worker_metadatas[rails_path] ||= QUEUE_CONFIG_PATHS.flat_map do |path|
           full_path = File.join(rails_path, path)
 
           File.exist?(full_path) ? YAML.load_file(full_path) : []
@@ -49,7 +33,7 @@ module Gitlab
       # rubocop:enable Gitlab/ModuleWithInstanceVariables
 
       def worker_queues(rails_path = Rails.root.to_s)
-        worker_names(all_queues(rails_path))
+        worker_names(worker_metadatas(rails_path))
       end
 
       def expand_queues(queues, all_queues = self.worker_queues)
@@ -62,13 +46,18 @@ module Gitlab
         end
       end
 
-      def query_workers(query_string, queues)
-        worker_names(queues.select(&query_string_to_lambda(query_string)))
+      def query_queues(query_string, worker_metadatas)
+        matcher = SidekiqConfig::WorkerMatcher.new(query_string)
+        selected_metadatas = worker_metadatas.select do |worker_metadata|
+          matcher.match?(worker_metadata)
+        end
+
+        worker_names(selected_metadatas)
       end
 
       def clear_memoization!
-        if instance_variable_defined?('@worker_queues')
-          remove_instance_variable('@worker_queues')
+        if instance_variable_defined?('@worker_metadatas')
+          remove_instance_variable('@worker_metadatas')
         end
       end
 
@@ -77,53 +66,6 @@ module Gitlab
       def worker_names(workers)
         workers.map { |queue| queue[:name] }
       end
-
-      def query_string_to_lambda(query_string)
-        or_clauses = query_string.split(QUERY_OR_OPERATOR).map do |and_clauses_string|
-          and_clauses_predicates = and_clauses_string.split(QUERY_AND_OPERATOR).map do |term|
-            predicate_for_term(term)
-          end
-
-          lambda { |worker| and_clauses_predicates.all? { |predicate| predicate.call(worker) } }
-        end
-
-        lambda { |worker| or_clauses.any? { |predicate| predicate.call(worker) } }
-      end
-
-      def predicate_for_term(term)
-        match = term.match(QUERY_TERM_REGEX)
-
-        raise InvalidTerm.new("Invalid term: #{term}") unless match
-
-        _, lhs, op, rhs = *match
-
-        predicate_for_op(op, predicate_factory(lhs, rhs.split(QUERY_CONCATENATE_OPERATOR)))
-      end
-
-      def predicate_for_op(op, predicate)
-        case op
-        when '='
-          predicate
-        when '!='
-          lambda { |worker| !predicate.call(worker) }
-        else
-          # This is unreachable because InvalidTerm will be raised instead, but
-          # keeping it allows to guard against that changing in future.
-          raise UnknownOperator.new("Unknown operator: #{op}")
-        end
-      end
-
-      def predicate_factory(lhs, values)
-        values_block = QUERY_PREDICATES[lhs.to_sym]
-
-        raise UnknownPredicate.new("Unknown predicate: #{lhs}") unless values_block
-
-        lambda do |queue|
-          comparator = Array(queue[lhs.to_sym]).to_set
-
-          values.map(&values_block).to_set.intersect?(comparator)
-        end
-      end
     end
   end
 end
diff --git a/lib/gitlab/sidekiq_config/worker_matcher.rb b/lib/gitlab/sidekiq_config/worker_matcher.rb
new file mode 100644
index 00000000000..fe5ac10c65a
--- /dev/null
+++ b/lib/gitlab/sidekiq_config/worker_matcher.rb
@@ -0,0 +1,86 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module SidekiqConfig
+    class WorkerMatcher
+      WILDCARD_MATCH = '*'
+      QUERY_OR_OPERATOR = '|'
+      QUERY_AND_OPERATOR = '&'
+      QUERY_CONCATENATE_OPERATOR = ','
+      QUERY_TERM_REGEX = %r{^(\w+)(!?=)([\w:#{QUERY_CONCATENATE_OPERATOR}]+)}.freeze
+
+      QUERY_PREDICATES = {
+        feature_category: :to_sym,
+        has_external_dependencies: lambda { |value| value == 'true' },
+        name: :to_s,
+        resource_boundary: :to_sym,
+        tags: :to_sym,
+        urgency: :to_sym
+      }.freeze
+
+      QueryError = Class.new(StandardError)
+      InvalidTerm = Class.new(QueryError)
+      UnknownOperator = Class.new(QueryError)
+      UnknownPredicate = Class.new(QueryError)
+
+      def initialize(query_string)
+        @match_lambda = query_string_to_lambda(query_string)
+      end
+
+      def match?(worker_metadata)
+        @match_lambda.call(worker_metadata)
+      end
+
+      private
+
+      def query_string_to_lambda(query_string)
+        return lambda { |_worker| true } if query_string.strip == WILDCARD_MATCH
+
+        or_clauses = query_string.split(QUERY_OR_OPERATOR).map do |and_clauses_string|
+          and_clauses_predicates = and_clauses_string.split(QUERY_AND_OPERATOR).map do |term|
+            predicate_for_term(term)
+          end
+
+          lambda { |worker| and_clauses_predicates.all? { |predicate| predicate.call(worker) } }
+        end
+
+        lambda { |worker| or_clauses.any? { |predicate| predicate.call(worker) } }
+      end
+
+      def predicate_for_term(term)
+        match = term.match(QUERY_TERM_REGEX)
+
+        raise InvalidTerm.new("Invalid term: #{term}") unless match
+
+        _, lhs, op, rhs = *match
+
+        predicate_for_op(op, predicate_factory(lhs, rhs.split(QUERY_CONCATENATE_OPERATOR)))
+      end
+
+      def predicate_for_op(op, predicate)
+        case op
+        when '='
+          predicate
+        when '!='
+          lambda { |worker| !predicate.call(worker) }
+        else
+          # This is unreachable because InvalidTerm will be raised instead, but
+          # keeping it allows to guard against that changing in future.
+          raise UnknownOperator.new("Unknown operator: #{op}")
+        end
+      end
+
+      def predicate_factory(lhs, values)
+        values_block = QUERY_PREDICATES[lhs.to_sym]
+
+        raise UnknownPredicate.new("Unknown predicate: #{lhs}") unless values_block
+
+        lambda do |queue|
+          comparator = Array(queue[lhs.to_sym]).to_set
+
+          values.map(&values_block).to_set.intersect?(comparator)
+        end
+      end
+    end
+  end
+end