Add latest changes from gitlab-org/gitlab@16-6-stable-eev16.6.0-rc42

5 files changed, 93 insertions, 22 deletions

diff --git a/lib/gitlab/redis/cluster_util.rb b/lib/gitlab/redis/cluster_util.rb
index 5f1f39b5237..9e307940de3 100644
--- a/lib/gitlab/redis/cluster_util.rb
+++ b/lib/gitlab/redis/cluster_util.rb
@@ -26,6 +26,15 @@ module Gitlab
           end
           expired_count
         end
+
+        # Redis cluster alternative to mget
+        def batch_get(keys, redis)
+          keys.each_slice(1000).flat_map do |subset|
+            Gitlab::Redis::CrossSlot::Pipeline.new(redis).pipelined do |pipeline|
+              subset.map { |key| pipeline.get(key) }
+            end
+          end
+        end
       end
     end
   end
diff --git a/lib/gitlab/redis/multi_store.rb b/lib/gitlab/redis/multi_store.rb
index bbe5a8add4b..6acbf83df24 100644
--- a/lib/gitlab/redis/multi_store.rb
+++ b/lib/gitlab/redis/multi_store.rb
@@ -63,8 +63,12 @@ module Gitlab
         hlen
         hmget
         hscan_each
+        llen
+        lrange
         mapped_hmget
         mget
+        pfcount
+        pttl
         scan
         scan_each
         scard
@@ -72,20 +76,32 @@ module Gitlab
         smembers
         sscan
         sscan_each
+        strlen
         ttl
+        type
+        zcard
+        zcount
+        zrange
+        zrangebyscore
+        zrevrange
         zscan_each
+        zscore
       ].freeze
 
       WRITE_COMMANDS = %i[
+        decr
         del
         eval
         expire
         flushdb
         hdel
+        hincrby
         hset
         incr
         incrby
         mapped_hmset
+        pfadd
+        pfmerge
         publish
         rpush
         sadd
@@ -93,8 +109,15 @@ module Gitlab
         set
         setex
         setnx
+        spop
         srem
+        srem?
         unlink
+        zadd
+        zpopmin
+        zrem
+        zremrangebyrank
+        zremrangebyscore
 
         memory
       ].freeze
@@ -254,11 +277,27 @@ module Gitlab
       #
       # Let's define it explicitly instead of propagating it to method_missing
       def close
-        if use_primary_and_secondary_stores?
-          [primary_store, secondary_store].map(&:close).first
+        if same_redis_store?
+          # if same_redis_store?, `use_primary_store_as_default?` returns false
+          # but we should avoid a feature-flag check in `.close` to avoid checking out
+          # an ActiveRecord connection during clean up.
+          secondary_store.close
         else
-          default_store.close
+          [primary_store, secondary_store].map(&:close).first
+        end
+      end
+
+      # blpop blocks until an element to be popped exist in the list or after a timeout.
+      def blpop(*args)
+        result = default_store.blpop(*args)
+        if !!result && use_primary_and_secondary_stores?
+          # special case to accommodate Gitlab::JobWaiter as blpop is only used in JobWaiter
+          # 1s should be sufficient wait time to account for delays between 1st and 2nd lpush
+          # https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/2520#note_1630893702
+          non_default_store.blpop(args.first, timeout: 1)
         end
+
+        result
       end
 
       private
@@ -380,7 +419,7 @@ module Gitlab
       end
 
       def redis_store?(store)
-        store.is_a?(::Redis) || store.is_a?(::Redis::Namespace)
+        store.is_a?(::Redis)
       end
 
       def validate_stores!
diff --git a/lib/gitlab/redis/pubsub.rb b/lib/gitlab/redis/pubsub.rb
deleted file mode 100644
index b5022f467a2..00000000000
--- a/lib/gitlab/redis/pubsub.rb
+++ /dev/null
@@ -1,13 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
-  module Redis
-    class Pubsub < ::Gitlab::Redis::Wrapper
-      class << self
-        def config_fallback
-          SharedState
-        end
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/redis/shared_state.rb b/lib/gitlab/redis/shared_state.rb
index fb3a143121b..d12d3e8c6aa 100644
--- a/lib/gitlab/redis/shared_state.rb
+++ b/lib/gitlab/redis/shared_state.rb
@@ -3,6 +3,12 @@
 module Gitlab
   module Redis
     class SharedState < ::Gitlab::Redis::Wrapper
+      def self.redis
+        primary_store = ::Redis.new(ClusterSharedState.params)
+        secondary_store = ::Redis.new(params)
+
+        MultiStore.new(primary_store, secondary_store, store_name)
+      end
     end
   end
 end
diff --git a/lib/gitlab/redis/wrapper.rb b/lib/gitlab/redis/wrapper.rb
index 2bcf4769b5a..d5470bc0016 100644
--- a/lib/gitlab/redis/wrapper.rb
+++ b/lib/gitlab/redis/wrapper.rb
@@ -19,7 +19,7 @@ module Gitlab
       InvalidPathError = Class.new(StandardError)
 
       class << self
-        delegate :params, :url, :store, to: :new
+        delegate :params, :url, :store, :encrypted_secrets, to: :new
 
         def with
           pool.with { |redis| yield redis }
@@ -110,6 +110,14 @@ module Gitlab
         raw_config_hash[:sentinels]
       end
 
+      def secret_file
+        if raw_config_hash[:secret_file].blank?
+          File.join(Settings.encrypted_settings['path'], 'redis.yaml.enc')
+        else
+          Settings.absolute(raw_config_hash[:secret_file])
+        end
+      end
+
       def sentinels?
         sentinels && !sentinels.empty?
       end
@@ -118,22 +126,44 @@ module Gitlab
         ::Redis::Store::Factory.create(redis_store_options.merge(extras))
       end
 
+      def encrypted_secrets
+        # In rake tasks, we have to populate the encrypted_secrets even if the
+        # file does not exist, as it is the job of one of those tasks to create
+        # the file. In other cases, like when being loaded as part of spinning
+        # up test environment via `scripts/setup-test-env`, we should gate on
+        # the presence of the specified secret file so that
+        # `Settings.encrypted`, which might not be loadable does not gets
+        # called.
+        Settings.encrypted(secret_file) if File.exist?(secret_file) || ::Gitlab::Runtime.rake?
+      end
+
       private
 
       def redis_store_options
         config = raw_config_hash
         config[:instrumentation_class] ||= self.class.instrumentation_class
 
-        result = if config[:cluster].present?
-                   config[:db] = 0 # Redis Cluster only supports db 0
-                   config
+        decrypted_config = parse_encrypted_config(config)
+
+        result = if decrypted_config[:cluster].present?
+                   decrypted_config[:db] = 0 # Redis Cluster only supports db 0
+                   decrypted_config
                  else
-                   parse_redis_url(config)
+                   parse_redis_url(decrypted_config)
                  end
 
         parse_client_tls_options(result)
       end
 
+      def parse_encrypted_config(encrypted_config)
+        encrypted_config.delete(:secret_file)
+
+        decrypted_secrets = encrypted_secrets&.config
+        encrypted_config.merge!(decrypted_secrets) if decrypted_secrets
+
+        encrypted_config
+      end
+
       def parse_redis_url(config)
         redis_url = config.delete(:url)
         redis_uri = URI.parse(redis_url)