diff options
Diffstat (limited to 'lib/gitlab/cache')
-rw-r--r-- | lib/gitlab/cache/import/caching.rb | 4 | ||||
-rw-r--r-- | lib/gitlab/cache/json_cache.rb | 123 | ||||
-rw-r--r-- | lib/gitlab/cache/json_caches/json_keyed.rb | 41 | ||||
-rw-r--r-- | lib/gitlab/cache/json_caches/redis_keyed.rb | 31 |
4 files changed, 197 insertions, 2 deletions
diff --git a/lib/gitlab/cache/import/caching.rb b/lib/gitlab/cache/import/caching.rb index 7fec6584ba3..8f2df29c320 100644 --- a/lib/gitlab/cache/import/caching.rb +++ b/lib/gitlab/cache/import/caching.rb @@ -162,13 +162,13 @@ module Gitlab def self.write_multiple(mapping, key_prefix: nil, timeout: TIMEOUT) with_redis do |redis| Gitlab::Instrumentation::RedisClusterValidator.allow_cross_slot_commands do - redis.pipelined do |multi| + Gitlab::Redis::CrossSlot::Pipeline.new(redis).pipelined do |pipeline| mapping.each do |raw_key, value| key = cache_key_for("#{key_prefix}#{raw_key}") validate_redis_value!(value) - multi.set(key, value, ex: timeout) + pipeline.set(key, value, ex: timeout) end end end diff --git a/lib/gitlab/cache/json_cache.rb b/lib/gitlab/cache/json_cache.rb new file mode 100644 index 00000000000..7450c7e540b --- /dev/null +++ b/lib/gitlab/cache/json_cache.rb @@ -0,0 +1,123 @@ +# frozen_string_literal: true + +module Gitlab + module Cache + class JsonCache + STRATEGY_KEY_COMPONENTS = { + revision: Gitlab.revision, + version: [Gitlab::VERSION, Rails.version] + }.freeze + + def initialize(options = {}) + @backend = options.fetch(:backend, Rails.cache) + @namespace = options.fetch(:namespace, nil) + @cache_key_strategy = options.fetch(:cache_key_strategy, :revision) + end + + def active? + if backend.respond_to?(:active?) + backend.active? + else + true + end + end + + def expire(key) + backend.delete(cache_key(key)) + end + + def read(key, klass = nil) + value = read_raw(key) + value = parse_value(value, klass) unless value.nil? + value + end + + def write(key, value, options = nil) + write_raw(key, value, options) + end + + def fetch(key, options = {}) + klass = options.delete(:as) + value = read(key, klass) + + return value unless value.nil? + + value = yield + + write(key, value, options) + + value + end + + private + + attr_reader :backend, :namespace, :cache_key_strategy + + def cache_key(key) + expanded_cache_key(key).compact.join(':').freeze + end + + def write_raw(_key, _value, _options) + raise NoMethodError + end + + def expanded_cache_key(_key) + raise NoMethodError + end + + def read_raw(_key) + raise NoMethodError + end + + def parse_value(value, klass) + case value + when Hash then parse_entry(value, klass) + when Array then parse_entries(value, klass) + else + value + end + end + + def parse_entry(raw, klass) + return unless valid_entry?(raw, klass) + return klass.new(raw) unless klass.ancestors.include?(ActiveRecord::Base) + + # When the cached value is a persisted instance of ActiveRecord::Base in + # some cases a relation can return an empty collection because scope.none! + # is being applied on ActiveRecord::Associations::CollectionAssociation#scope + # when the new_record? method incorrectly returns false. + # + # See https://gitlab.com/gitlab-org/gitlab/issues/9903#note_145329964 + klass.allocate.init_with(encode_for(klass, raw)) + end + + def encode_for(klass, raw) + # We have models that leave out some fields from the JSON export for + # security reasons, e.g. models that include the CacheMarkdownField. + # The ActiveRecord::AttributeSet we build from raw does know about + # these columns so we need manually set them. + missing_attributes = (klass.columns.map(&:name) - raw.keys) + missing_attributes.each { |column| raw[column] = nil } + + coder = {} + klass.new(raw).encode_with(coder) + coder["new_record"] = new_record?(raw, klass) + coder + end + + def new_record?(raw, klass) + raw.fetch(klass.primary_key, nil).blank? + end + + def valid_entry?(raw, klass) + return false unless klass && raw.is_a?(Hash) + + (raw.keys - klass.attribute_names).empty? + end + + def parse_entries(values, klass) + values.filter_map { |value| parse_entry(value, klass) } + end + end + end +end diff --git a/lib/gitlab/cache/json_caches/json_keyed.rb b/lib/gitlab/cache/json_caches/json_keyed.rb new file mode 100644 index 00000000000..701a49c23de --- /dev/null +++ b/lib/gitlab/cache/json_caches/json_keyed.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +module Gitlab + module Cache + module JsonCaches + class JsonKeyed < JsonCache + private + + def expanded_cache_key(key) + [namespace, key] + end + + def write_raw(key, value, options = nil) + raw_value = {} + + begin + read_value = backend.read(cache_key(key)) + read_value = Gitlab::Json.parse(read_value.to_s) unless read_value.nil? + raw_value = read_value if read_value.is_a?(Hash) + rescue JSON::ParserError + end + + raw_value[strategy_key_component] = value + backend.write(cache_key(key), raw_value.to_json, options) + end + + def read_raw(key) + value = backend.read(cache_key(key)) + value = Gitlab::Json.parse(value.to_s) unless value.nil? + value[strategy_key_component] if value.is_a?(Hash) + rescue JSON::ParserError + nil + end + + def strategy_key_component + Array.wrap(STRATEGY_KEY_COMPONENTS.fetch(cache_key_strategy)).compact.join(':').freeze + end + end + end + end +end diff --git a/lib/gitlab/cache/json_caches/redis_keyed.rb b/lib/gitlab/cache/json_caches/redis_keyed.rb new file mode 100644 index 00000000000..92709adef63 --- /dev/null +++ b/lib/gitlab/cache/json_caches/redis_keyed.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +module Gitlab + module Cache + module JsonCaches + class RedisKeyed < JsonCache + private + + def expanded_cache_key(key) + [namespace, key, *strategy_key_component] + end + + def write_raw(key, value, options) + backend.write(cache_key(key), value.to_json, options) + end + + def read_raw(key) + value = backend.read(cache_key(key)) + value = Gitlab::Json.parse(value.to_s) unless value.nil? + value + rescue JSON::ParserError + nil + end + + def strategy_key_component + STRATEGY_KEY_COMPONENTS.fetch(cache_key_strategy) + end + end + end + end +end |