diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2023-11-14 11:41:52 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2023-11-14 11:41:52 +0300 |
commit | 585826cb22ecea5998a2c2a4675735c94bdeedac (patch) | |
tree | 5b05f0b30d33cef48963609e8a18a4dff260eab3 /lib/gitlab/redis | |
parent | df221d036e5d0c6c0ee4d55b9c97f481ee05dee8 (diff) |
Add latest changes from gitlab-org/gitlab@16-6-stable-eev16.6.0-rc42
Diffstat (limited to 'lib/gitlab/redis')
-rw-r--r-- | lib/gitlab/redis/cluster_util.rb | 9 | ||||
-rw-r--r-- | lib/gitlab/redis/multi_store.rb | 47 | ||||
-rw-r--r-- | lib/gitlab/redis/pubsub.rb | 13 | ||||
-rw-r--r-- | lib/gitlab/redis/shared_state.rb | 6 | ||||
-rw-r--r-- | lib/gitlab/redis/wrapper.rb | 40 |
5 files changed, 93 insertions, 22 deletions
diff --git a/lib/gitlab/redis/cluster_util.rb b/lib/gitlab/redis/cluster_util.rb index 5f1f39b5237..9e307940de3 100644 --- a/lib/gitlab/redis/cluster_util.rb +++ b/lib/gitlab/redis/cluster_util.rb @@ -26,6 +26,15 @@ module Gitlab end expired_count end + + # Redis cluster alternative to mget + def batch_get(keys, redis) + keys.each_slice(1000).flat_map do |subset| + Gitlab::Redis::CrossSlot::Pipeline.new(redis).pipelined do |pipeline| + subset.map { |key| pipeline.get(key) } + end + end + end end end end diff --git a/lib/gitlab/redis/multi_store.rb b/lib/gitlab/redis/multi_store.rb index bbe5a8add4b..6acbf83df24 100644 --- a/lib/gitlab/redis/multi_store.rb +++ b/lib/gitlab/redis/multi_store.rb @@ -63,8 +63,12 @@ module Gitlab hlen hmget hscan_each + llen + lrange mapped_hmget mget + pfcount + pttl scan scan_each scard @@ -72,20 +76,32 @@ module Gitlab smembers sscan sscan_each + strlen ttl + type + zcard + zcount + zrange + zrangebyscore + zrevrange zscan_each + zscore ].freeze WRITE_COMMANDS = %i[ + decr del eval expire flushdb hdel + hincrby hset incr incrby mapped_hmset + pfadd + pfmerge publish rpush sadd @@ -93,8 +109,15 @@ module Gitlab set setex setnx + spop srem + srem? unlink + zadd + zpopmin + zrem + zremrangebyrank + zremrangebyscore memory ].freeze @@ -254,11 +277,27 @@ module Gitlab # # Let's define it explicitly instead of propagating it to method_missing def close - if use_primary_and_secondary_stores? - [primary_store, secondary_store].map(&:close).first + if same_redis_store? + # if same_redis_store?, `use_primary_store_as_default?` returns false + # but we should avoid a feature-flag check in `.close` to avoid checking out + # an ActiveRecord connection during clean up. + secondary_store.close else - default_store.close + [primary_store, secondary_store].map(&:close).first + end + end + + # blpop blocks until an element to be popped exist in the list or after a timeout. + def blpop(*args) + result = default_store.blpop(*args) + if !!result && use_primary_and_secondary_stores? + # special case to accommodate Gitlab::JobWaiter as blpop is only used in JobWaiter + # 1s should be sufficient wait time to account for delays between 1st and 2nd lpush + # https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/2520#note_1630893702 + non_default_store.blpop(args.first, timeout: 1) end + + result end private @@ -380,7 +419,7 @@ module Gitlab end def redis_store?(store) - store.is_a?(::Redis) || store.is_a?(::Redis::Namespace) + store.is_a?(::Redis) end def validate_stores! diff --git a/lib/gitlab/redis/pubsub.rb b/lib/gitlab/redis/pubsub.rb deleted file mode 100644 index b5022f467a2..00000000000 --- a/lib/gitlab/redis/pubsub.rb +++ /dev/null @@ -1,13 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Redis - class Pubsub < ::Gitlab::Redis::Wrapper - class << self - def config_fallback - SharedState - end - end - end - end -end diff --git a/lib/gitlab/redis/shared_state.rb b/lib/gitlab/redis/shared_state.rb index fb3a143121b..d12d3e8c6aa 100644 --- a/lib/gitlab/redis/shared_state.rb +++ b/lib/gitlab/redis/shared_state.rb @@ -3,6 +3,12 @@ module Gitlab module Redis class SharedState < ::Gitlab::Redis::Wrapper + def self.redis + primary_store = ::Redis.new(ClusterSharedState.params) + secondary_store = ::Redis.new(params) + + MultiStore.new(primary_store, secondary_store, store_name) + end end end end diff --git a/lib/gitlab/redis/wrapper.rb b/lib/gitlab/redis/wrapper.rb index 2bcf4769b5a..d5470bc0016 100644 --- a/lib/gitlab/redis/wrapper.rb +++ b/lib/gitlab/redis/wrapper.rb @@ -19,7 +19,7 @@ module Gitlab InvalidPathError = Class.new(StandardError) class << self - delegate :params, :url, :store, to: :new + delegate :params, :url, :store, :encrypted_secrets, to: :new def with pool.with { |redis| yield redis } @@ -110,6 +110,14 @@ module Gitlab raw_config_hash[:sentinels] end + def secret_file + if raw_config_hash[:secret_file].blank? + File.join(Settings.encrypted_settings['path'], 'redis.yaml.enc') + else + Settings.absolute(raw_config_hash[:secret_file]) + end + end + def sentinels? sentinels && !sentinels.empty? end @@ -118,22 +126,44 @@ module Gitlab ::Redis::Store::Factory.create(redis_store_options.merge(extras)) end + def encrypted_secrets + # In rake tasks, we have to populate the encrypted_secrets even if the + # file does not exist, as it is the job of one of those tasks to create + # the file. In other cases, like when being loaded as part of spinning + # up test environment via `scripts/setup-test-env`, we should gate on + # the presence of the specified secret file so that + # `Settings.encrypted`, which might not be loadable does not gets + # called. + Settings.encrypted(secret_file) if File.exist?(secret_file) || ::Gitlab::Runtime.rake? + end + private def redis_store_options config = raw_config_hash config[:instrumentation_class] ||= self.class.instrumentation_class - result = if config[:cluster].present? - config[:db] = 0 # Redis Cluster only supports db 0 - config + decrypted_config = parse_encrypted_config(config) + + result = if decrypted_config[:cluster].present? + decrypted_config[:db] = 0 # Redis Cluster only supports db 0 + decrypted_config else - parse_redis_url(config) + parse_redis_url(decrypted_config) end parse_client_tls_options(result) end + def parse_encrypted_config(encrypted_config) + encrypted_config.delete(:secret_file) + + decrypted_secrets = encrypted_secrets&.config + encrypted_config.merge!(decrypted_secrets) if decrypted_secrets + + encrypted_config + end + def parse_redis_url(config) redis_url = config.delete(:url) redis_uri = URI.parse(redis_url) |