Add latest changes from gitlab-org/gitlab@15-8-stable-eev15.8.0-rc42

3 files changed, 151 insertions, 60 deletions

diff --git a/lib/gitlab/redis/multi_store.rb b/lib/gitlab/redis/multi_store.rb
index 4f58bee49d0..aa8f390ac10 100644
--- a/lib/gitlab/redis/multi_store.rb
+++ b/lib/gitlab/redis/multi_store.rb
@@ -26,7 +26,7 @@ module Gitlab
 
       class MethodMissingError < StandardError
         def message
-          'Method missing. Falling back to execute method on the redis secondary store.'
+          'Method missing. Falling back to execute method on the redis default store in Rails.env.production.'
         end
       end
 
@@ -36,31 +36,64 @@ module Gitlab
       FAILED_TO_WRITE_ERROR_MESSAGE = 'Failed to write to the redis primary_store.'
       FAILED_TO_RUN_PIPELINE = 'Failed to execute pipeline on the redis primary_store.'
 
-      SKIP_LOG_METHOD_MISSING_FOR_COMMANDS = %i(info).freeze
+      SKIP_LOG_METHOD_MISSING_FOR_COMMANDS = %i[info].freeze
 
-      READ_COMMANDS = %i(
-        get
-        mget
-        smembers
-        scard
-      ).freeze
-
-      WRITE_COMMANDS = %i(
-        set
-        setnx
-        setex
-        sadd
-        srem
+      # For ENUMERATOR_CACHE_HIT_VALIDATOR and READ_CACHE_HIT_VALIDATOR,
+      # we define procs to validate cache hit. The only other acceptable value is nil,
+      # in the case of errors being raised.
+      #
+      # If a command has no empty response, set ->(val) { true }
+      #
+      # Ref: https://www.rubydoc.info/github/redis/redis-rb/Redis/Commands
+      #
+      ENUMERATOR_CACHE_HIT_VALIDATOR = {
+        scan_each: ->(val) { val.is_a?(Enumerator) && !val.first.nil? },
+        hscan_each: ->(val) { val.is_a?(Enumerator) && !val.first.nil? },
+        sscan_each: ->(val) { val.is_a?(Enumerator) && !val.first.nil? },
+        zscan_each: ->(val) { val.is_a?(Enumerator) && !val.first.nil? }
+      }.freeze
+
+      READ_CACHE_HIT_VALIDATOR = {
+        exists: ->(val) { val != 0 },
+        exists?: ->(val) { val },
+        get: ->(val) { !val.nil? },
+        hexists: ->(val) { val },
+        hget: ->(val) { !val.nil? },
+        hgetall:  ->(val) { val.is_a?(Hash) && !val.empty? },
+        hlen: ->(val) { val != 0 },
+        hmget: ->(val) { val.is_a?(Array) && !val.compact.empty? },
+        mapped_hmget: ->(val) { val.is_a?(Hash) && !val.compact.empty? },
+        mget: ->(val) { val.is_a?(Array) && !val.compact.empty? },
+        scard: ->(val) { val != 0 },
+        sismember: ->(val) { val },
+        smembers: ->(val) { val.is_a?(Array) && !val.empty? },
+        sscan: ->(val) { val != ['0', []] },
+        ttl: ->(val) { val != 0 && val != -2 }
+      }.freeze
+
+      WRITE_COMMANDS = %i[
         del
+        eval
+        expire
         flushdb
+        hdel
+        hset
+        incr
+        incrby
+        mapped_hmset
         rpush
-        eval
-      ).freeze
+        sadd
+        set
+        setex
+        setnx
+        srem
+        unlink
+      ].freeze
 
-      PIPELINED_COMMANDS = %i(
+      PIPELINED_COMMANDS = %i[
         pipelined
         multi
-      ).freeze
+      ].freeze
 
       # To transition between two Redis store, `primary_store` should be the target store,
       # and `secondary_store` should be the current store. Transition is controlled with feature flags:
@@ -81,12 +114,12 @@ module Gitlab
       end
 
       # rubocop:disable GitlabSecurity/PublicSend
-      READ_COMMANDS.each do |name|
-        define_method(name) do |*args, &block|
+      READ_CACHE_HIT_VALIDATOR.each_key do |name|
+        define_method(name) do |*args, **kwargs, &block|
           if use_primary_and_secondary_stores?
-            read_command(name, *args, &block)
+            read_command(name, *args, **kwargs, &block)
           else
-            default_store.send(name, *args, &block)
+            default_store.send(name, *args, **kwargs, &block)
           end
         end
       end
@@ -101,6 +134,20 @@ module Gitlab
         end
       end
 
+      ENUMERATOR_CACHE_HIT_VALIDATOR.each_key do |name|
+        define_method(name) do |*args, **kwargs, &block|
+          enumerator = if use_primary_and_secondary_stores?
+                         read_command(name, *args, **kwargs)
+                       else
+                         default_store.send(name, *args, **kwargs)
+                       end
+
+          return enumerator if block.nil?
+
+          enumerator.each(&block)
+        end
+      end
+
       PIPELINED_COMMANDS.each do |name|
         define_method(name) do |*args, **kwargs, &block|
           if use_primary_and_secondary_stores?
@@ -170,12 +217,23 @@ module Gitlab
           extra.merge(command_name: command_name, instance_name: instance_name))
       end
 
+      def ping(message = nil)
+        if use_primary_and_secondary_stores?
+          # Both stores have to response success for the ping to be considered success.
+          # We assume both stores cannot return different responses (only both "PONG" or both echo the message).
+          # If either store is not reachable, an Error will be raised anyway thus taking any response works.
+          [primary_store, secondary_store].map { |store| store.ping(message) }.first
+        else
+          default_store.ping(message)
+        end
+      end
+
       private
 
       # @return [Boolean]
       def feature_enabled?(prefix)
         feature_table_exists? &&
-          Feature.enabled?("#{prefix}_#{instance_name.underscore}") &&
+          Feature.enabled?("#{prefix}_#{instance_name.underscore}") && # rubocop:disable Cop/FeatureFlagUsage
           !same_redis_store?
       end
 
@@ -193,15 +251,17 @@ module Gitlab
       def log_method_missing(command_name, *_args)
         return if SKIP_LOG_METHOD_MISSING_FOR_COMMANDS.include?(command_name)
 
+        raise MethodMissingError if Rails.env.test? || Rails.env.development?
+
         log_error(MethodMissingError.new, command_name)
         increment_method_missing_count(command_name)
       end
 
-      def read_command(command_name, *args, &block)
+      def read_command(command_name, *args, **kwargs, &block)
         if @instance
-          send_command(@instance, command_name, *args, &block)
+          send_command(@instance, command_name, *args, **kwargs, &block)
         else
-          read_one_with_fallback(command_name, *args, &block)
+          read_one_with_fallback(command_name, *args, **kwargs, &block)
         end
       end
 
@@ -213,19 +273,28 @@ module Gitlab
         end
       end
 
-      def read_one_with_fallback(command_name, *args, &block)
+      def read_one_with_fallback(command_name, *args, **kwargs, &block)
         begin
-          value = send_command(primary_store, command_name, *args, &block)
+          value = send_command(primary_store, command_name, *args, **kwargs, &block)
         rescue StandardError => e
           log_error(e, command_name,
             multi_store_error_message: FAILED_TO_READ_ERROR_MESSAGE)
         end
 
-        value || fallback_read(command_name, *args, &block)
+        return value if cache_hit?(command_name, value)
+
+        fallback_read(command_name, *args, **kwargs, &block)
+      end
+
+      def cache_hit?(command, value)
+        validator = READ_CACHE_HIT_VALIDATOR[command] || ENUMERATOR_CACHE_HIT_VALIDATOR[command]
+        return false unless validator
+
+        !value.nil? && validator.call(value)
       end
 
-      def fallback_read(command_name, *args, &block)
-        value = send_command(secondary_store, command_name, *args, &block)
+      def fallback_read(command_name, *args, **kwargs, &block)
+        value = send_command(secondary_store, command_name, *args, **kwargs, &block)
 
         if value
           log_error(ReadFromPrimaryError.new, command_name)
diff --git a/lib/gitlab/redis/repository_cache.rb b/lib/gitlab/redis/repository_cache.rb
new file mode 100644
index 00000000000..8bfbfcfea60
--- /dev/null
+++ b/lib/gitlab/redis/repository_cache.rb
@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Redis
+    class RepositoryCache < ::Gitlab::Redis::Wrapper
+      class << self
+        # The data we store on RepositoryCache used to be stored on Cache.
+        def config_fallback
+          Cache
+        end
+
+        def cache_store
+          @cache_store ||= ActiveSupport::Cache::RedisCacheStore.new(
+            redis: pool,
+            compress: Gitlab::Utils.to_boolean(ENV.fetch('ENABLE_REDIS_CACHE_COMPRESSION', '1')),
+            namespace: Cache::CACHE_NAMESPACE,
+            # Cache should not grow forever
+            expires_in: ENV.fetch('GITLAB_RAILS_CACHE_DEFAULT_TTL_SECONDS', 8.hours).to_i
+          )
+        end
+
+        private
+
+        def redis
+          primary_store = ::Redis.new(params)
+          secondary_store = ::Redis.new(config_fallback.params)
+
+          MultiStore.new(primary_store, secondary_store, store_name)
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/redis/wrapper.rb b/lib/gitlab/redis/wrapper.rb
index 0e5389dc995..e5e1e1d4165 100644
--- a/lib/gitlab/redis/wrapper.rb
+++ b/lib/gitlab/redis/wrapper.rb
@@ -41,21 +41,6 @@ module Gitlab
           size
         end
 
-        def _raw_config
-          return @_raw_config if defined?(@_raw_config)
-
-          @_raw_config =
-            begin
-              if filename = config_file_name
-                ERB.new(File.read(filename)).result.freeze
-              else
-                false
-              end
-            rescue Errno::ENOENT
-              false
-            end
-        end
-
         def config_file_path(filename)
           path = File.join(rails_root, 'config', filename)
           return path if File.file?(path)
@@ -67,10 +52,6 @@ module Gitlab
           File.expand_path('../../..', __dir__)
         end
 
-        def config_fallback?
-          config_file_name == config_fallback&.config_file_name
-        end
-
         def config_file_name
           [
             # Instance specific config sources:
@@ -91,6 +72,10 @@ module Gitlab
           ].compact.first
         end
 
+        def redis_yml_path
+          File.join(rails_root, 'config/redis.yml')
+        end
+
         def store_name
           name.demodulize
         end
@@ -212,16 +197,20 @@ module Gitlab
       end
 
       def fetch_config
-        return false unless self.class._raw_config
-
-        yaml = YAML.safe_load(self.class._raw_config, aliases: true)
+        redis_yml = read_yaml(self.class.redis_yml_path).fetch(@rails_env, {})
+        instance_config_yml = read_yaml(self.class.config_file_name)[@rails_env]
+
+        [
+          redis_yml[self.class.store_name.underscore],
+          instance_config_yml,
+          self.class.config_fallback && redis_yml[self.class.config_fallback.store_name.underscore]
+        ].compact.first
+      end
 
-        # If the file has content but it's invalid YAML, `load` returns false
-        if yaml
-          yaml.fetch(@rails_env, false)
-        else
-          false
-        end
+      def read_yaml(path)
+        YAML.safe_load(ERB.new(File.read(path.to_s)).result, aliases: true) || {}
+      rescue Errno::ENOENT
+        {}
       end
     end
   end