Add latest changes from gitlab-org/gitlab@16-1-stable-eev16.1.0-rc42

4 files changed, 197 insertions, 2 deletions

diff --git a/lib/gitlab/cache/import/caching.rb b/lib/gitlab/cache/import/caching.rb
index 7fec6584ba3..8f2df29c320 100644
--- a/lib/gitlab/cache/import/caching.rb
+++ b/lib/gitlab/cache/import/caching.rb
@@ -162,13 +162,13 @@ module Gitlab
         def self.write_multiple(mapping, key_prefix: nil, timeout: TIMEOUT)
           with_redis do |redis|
             Gitlab::Instrumentation::RedisClusterValidator.allow_cross_slot_commands do
-              redis.pipelined do |multi|
+              Gitlab::Redis::CrossSlot::Pipeline.new(redis).pipelined do |pipeline|
                 mapping.each do |raw_key, value|
                   key = cache_key_for("#{key_prefix}#{raw_key}")
 
                   validate_redis_value!(value)
 
-                  multi.set(key, value, ex: timeout)
+                  pipeline.set(key, value, ex: timeout)
                 end
               end
             end
diff --git a/lib/gitlab/cache/json_cache.rb b/lib/gitlab/cache/json_cache.rb
new file mode 100644
index 00000000000..7450c7e540b
--- /dev/null
+++ b/lib/gitlab/cache/json_cache.rb
@@ -0,0 +1,123 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Cache
+    class JsonCache
+      STRATEGY_KEY_COMPONENTS = {
+        revision: Gitlab.revision,
+        version: [Gitlab::VERSION, Rails.version]
+      }.freeze
+
+      def initialize(options = {})
+        @backend = options.fetch(:backend, Rails.cache)
+        @namespace = options.fetch(:namespace, nil)
+        @cache_key_strategy = options.fetch(:cache_key_strategy, :revision)
+      end
+
+      def active?
+        if backend.respond_to?(:active?)
+          backend.active?
+        else
+          true
+        end
+      end
+
+      def expire(key)
+        backend.delete(cache_key(key))
+      end
+
+      def read(key, klass = nil)
+        value = read_raw(key)
+        value = parse_value(value, klass) unless value.nil?
+        value
+      end
+
+      def write(key, value, options = nil)
+        write_raw(key, value, options)
+      end
+
+      def fetch(key, options = {})
+        klass = options.delete(:as)
+        value = read(key, klass)
+
+        return value unless value.nil?
+
+        value = yield
+
+        write(key, value, options)
+
+        value
+      end
+
+      private
+
+      attr_reader :backend, :namespace, :cache_key_strategy
+
+      def cache_key(key)
+        expanded_cache_key(key).compact.join(':').freeze
+      end
+
+      def write_raw(_key, _value, _options)
+        raise NoMethodError
+      end
+
+      def expanded_cache_key(_key)
+        raise NoMethodError
+      end
+
+      def read_raw(_key)
+        raise NoMethodError
+      end
+
+      def parse_value(value, klass)
+        case value
+        when Hash then parse_entry(value, klass)
+        when Array then parse_entries(value, klass)
+        else
+          value
+        end
+      end
+
+      def parse_entry(raw, klass)
+        return unless valid_entry?(raw, klass)
+        return klass.new(raw) unless klass.ancestors.include?(ActiveRecord::Base)
+
+        # When the cached value is a persisted instance of ActiveRecord::Base in
+        # some cases a relation can return an empty collection because scope.none!
+        # is being applied on ActiveRecord::Associations::CollectionAssociation#scope
+        # when the new_record? method incorrectly returns false.
+        #
+        # See https://gitlab.com/gitlab-org/gitlab/issues/9903#note_145329964
+        klass.allocate.init_with(encode_for(klass, raw))
+      end
+
+      def encode_for(klass, raw)
+        # We have models that leave out some fields from the JSON export for
+        # security reasons, e.g. models that include the CacheMarkdownField.
+        # The ActiveRecord::AttributeSet we build from raw does know about
+        # these columns so we need manually set them.
+        missing_attributes = (klass.columns.map(&:name) - raw.keys)
+        missing_attributes.each { |column| raw[column] = nil }
+
+        coder = {}
+        klass.new(raw).encode_with(coder)
+        coder["new_record"] = new_record?(raw, klass)
+        coder
+      end
+
+      def new_record?(raw, klass)
+        raw.fetch(klass.primary_key, nil).blank?
+      end
+
+      def valid_entry?(raw, klass)
+        return false unless klass && raw.is_a?(Hash)
+
+        (raw.keys - klass.attribute_names).empty?
+      end
+
+      def parse_entries(values, klass)
+        values.filter_map { |value| parse_entry(value, klass) }
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/cache/json_caches/json_keyed.rb b/lib/gitlab/cache/json_caches/json_keyed.rb
new file mode 100644
index 00000000000..701a49c23de
--- /dev/null
+++ b/lib/gitlab/cache/json_caches/json_keyed.rb
@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Cache
+    module JsonCaches
+      class JsonKeyed < JsonCache
+        private
+
+        def expanded_cache_key(key)
+          [namespace, key]
+        end
+
+        def write_raw(key, value, options = nil)
+          raw_value = {}
+
+          begin
+            read_value = backend.read(cache_key(key))
+            read_value = Gitlab::Json.parse(read_value.to_s) unless read_value.nil?
+            raw_value = read_value if read_value.is_a?(Hash)
+          rescue JSON::ParserError
+          end
+
+          raw_value[strategy_key_component] = value
+          backend.write(cache_key(key), raw_value.to_json, options)
+        end
+
+        def read_raw(key)
+          value = backend.read(cache_key(key))
+          value = Gitlab::Json.parse(value.to_s) unless value.nil?
+          value[strategy_key_component] if value.is_a?(Hash)
+        rescue JSON::ParserError
+          nil
+        end
+
+        def strategy_key_component
+          Array.wrap(STRATEGY_KEY_COMPONENTS.fetch(cache_key_strategy)).compact.join(':').freeze
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/cache/json_caches/redis_keyed.rb b/lib/gitlab/cache/json_caches/redis_keyed.rb
new file mode 100644
index 00000000000..92709adef63
--- /dev/null
+++ b/lib/gitlab/cache/json_caches/redis_keyed.rb
@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Cache
+    module JsonCaches
+      class RedisKeyed < JsonCache
+        private
+
+        def expanded_cache_key(key)
+          [namespace, key, *strategy_key_component]
+        end
+
+        def write_raw(key, value, options)
+          backend.write(cache_key(key), value.to_json, options)
+        end
+
+        def read_raw(key)
+          value = backend.read(cache_key(key))
+          value = Gitlab::Json.parse(value.to_s) unless value.nil?
+          value
+        rescue JSON::ParserError
+          nil
+        end
+
+        def strategy_key_component
+          STRATEGY_KEY_COMPONENTS.fetch(cache_key_strategy)
+        end
+      end
+    end
+  end
+end