diff options
Diffstat (limited to 'lib/gitlab')
| -rw-r--r-- | lib/gitlab/anonymous_session.rb | 10 | ||||
| -rw-r--r-- | lib/gitlab/application_context.rb | 7 | ||||
| -rw-r--r-- | lib/gitlab/database/partitioning/monthly_strategy.rb | 4 | ||||
| -rw-r--r-- | lib/gitlab/database/partitioning/partition_manager.rb | 1 | ||||
| -rw-r--r-- | lib/gitlab/database/partitioning/single_numeric_list_partition.rb | 76 | ||||
| -rw-r--r-- | lib/gitlab/database/partitioning/sliding_list_strategy.rb | 75 | ||||
| -rw-r--r-- | lib/gitlab/redis/multi_store.rb | 2 | ||||
| -rw-r--r-- | lib/gitlab/redis/sessions.rb | 42 | ||||
| -rw-r--r-- | lib/gitlab/redis/sessions_store_helper.rb | 27 | ||||
| -rw-r--r-- | lib/gitlab/redis/shared_state.rb | 4 | ||||
| -rw-r--r-- | lib/gitlab/redis/wrapper.rb | 12 | ||||
| -rw-r--r-- | lib/gitlab/sidekiq_config.rb | 11 |
12 files changed, 252 insertions, 19 deletions
diff --git a/lib/gitlab/anonymous_session.rb b/lib/gitlab/anonymous_session.rb index 911825eef3a..fc78b64830a 100644 --- a/lib/gitlab/anonymous_session.rb +++ b/lib/gitlab/anonymous_session.rb @@ -2,12 +2,14 @@ module Gitlab class AnonymousSession + include ::Gitlab::Redis::SessionsStoreHelper + def initialize(remote_ip) @remote_ip = remote_ip end def count_session_ip - Gitlab::Redis::SharedState.with do |redis| + redis_store_class.with do |redis| redis.pipelined do redis.incr(session_lookup_name) redis.expire(session_lookup_name, 24.hours) @@ -16,13 +18,13 @@ module Gitlab end def session_count - Gitlab::Redis::SharedState.with do |redis| + redis_store_class.with do |redis| redis.get(session_lookup_name).to_i end end def cleanup_session_per_ip_count - Gitlab::Redis::SharedState.with do |redis| + redis_store_class.with do |redis| redis.del(session_lookup_name) end end @@ -32,7 +34,7 @@ module Gitlab attr_reader :remote_ip def session_lookup_name - @session_lookup_name ||= "#{Gitlab::Redis::SharedState::IP_SESSIONS_LOOKUP_NAMESPACE}:#{remote_ip}" + @session_lookup_name ||= "#{Gitlab::Redis::Sessions::IP_SESSIONS_LOOKUP_NAMESPACE}:#{remote_ip}" end end end diff --git a/lib/gitlab/application_context.rb b/lib/gitlab/application_context.rb index aa33f56582b..c3415c45b28 100644 --- a/lib/gitlab/application_context.rb +++ b/lib/gitlab/application_context.rb @@ -124,11 +124,8 @@ module Gitlab strong_memoize(:runner_project) do next unless runner&.project_type? - projects = ::Gitlab::Database.allow_cross_joins_across_databases(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/342147') do - runner.projects.take(2) # rubocop: disable CodeReuse/ActiveRecord - end - - projects.first if projects.one? + runner_projects = runner.runner_projects.take(2) # rubocop: disable CodeReuse/ActiveRecord + runner_projects.first.project if runner_projects.one? end end diff --git a/lib/gitlab/database/partitioning/monthly_strategy.rb b/lib/gitlab/database/partitioning/monthly_strategy.rb index c93e775d7ed..9c8cccb3dc6 100644 --- a/lib/gitlab/database/partitioning/monthly_strategy.rb +++ b/lib/gitlab/database/partitioning/monthly_strategy.rb @@ -36,6 +36,10 @@ module Gitlab partitions end + def after_adding_partitions + # No-op, required by the partition manager + end + private def desired_partitions diff --git a/lib/gitlab/database/partitioning/partition_manager.rb b/lib/gitlab/database/partitioning/partition_manager.rb index 8742c0ff166..3a4e120651e 100644 --- a/lib/gitlab/database/partitioning/partition_manager.rb +++ b/lib/gitlab/database/partitioning/partition_manager.rb @@ -73,6 +73,7 @@ module Gitlab partition_name: partition.partition_name, table_name: partition.table) end + model.partitioning_strategy.after_adding_partitions end end end diff --git a/lib/gitlab/database/partitioning/single_numeric_list_partition.rb b/lib/gitlab/database/partitioning/single_numeric_list_partition.rb new file mode 100644 index 00000000000..23ac73a0e53 --- /dev/null +++ b/lib/gitlab/database/partitioning/single_numeric_list_partition.rb @@ -0,0 +1,76 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module Partitioning + class SingleNumericListPartition + include Comparable + + def self.from_sql(table, partition_name, definition) + # A list partition can support multiple values, but we only support a single number + matches = definition.match(/FOR VALUES IN \('(?<value>\d+)'\)/) + + raise ArgumentError, 'Unknown partition definition' unless matches + + value = Integer(matches[:value]) + + new(table, value, partition_name: partition_name) + end + + attr_reader :table, :value + + def initialize(table, value, partition_name: nil ) + @table = table + @value = value + @partition_name = partition_name + end + + def partition_name + @partition_name || "#{table}_#{value}" + end + + def to_sql + <<~SQL + CREATE TABLE IF NOT EXISTS #{fully_qualified_partition} + PARTITION OF #{conn.quote_table_name(table)} + FOR VALUES IN (#{conn.quote(value)}) + SQL + end + + def to_detach_sql + <<~SQL + ALTER TABLE #{conn.quote_table_name(table)} + DETACH PARTITION #{fully_qualified_partition} + SQL + end + + def ==(other) + table == other.table && + partition_name == other.partition_name && + value == other.value + end + alias_method :eql?, :== + + def hash + [table, partition_name, value].hash + end + + def <=>(other) + return if table != other.table + + value <=> other.value + end + + private + + def fully_qualified_partition + "%s.%s" % [conn.quote_table_name(Gitlab::Database::DYNAMIC_PARTITIONS_SCHEMA), conn.quote_table_name(partition_name)] + end + + def conn + @conn ||= Gitlab::Database::SharedModel.connection + end + end + end + end +end diff --git a/lib/gitlab/database/partitioning/sliding_list_strategy.rb b/lib/gitlab/database/partitioning/sliding_list_strategy.rb new file mode 100644 index 00000000000..21b86b43ae7 --- /dev/null +++ b/lib/gitlab/database/partitioning/sliding_list_strategy.rb @@ -0,0 +1,75 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module Partitioning + class SlidingListStrategy + attr_reader :model, :partitioning_key, :next_partition_if, :detach_partition_if + + delegate :table_name, to: :model + + def initialize(model, partitioning_key, next_partition_if:, detach_partition_if:) + @model = model + @partitioning_key = partitioning_key + @next_partition_if = next_partition_if + @detach_partition_if = detach_partition_if + + ensure_partitioning_column_ignored! + end + + def current_partitions + Gitlab::Database::PostgresPartition.for_parent_table(table_name).map do |partition| + SingleNumericListPartition.from_sql(table_name, partition.name, partition.condition) + end.sort + end + + def missing_partitions + if no_partitions_exist? + [initial_partition] + elsif next_partition_if.call(active_partition.value) + [next_partition] + else + [] + end + end + + def initial_partition + SingleNumericListPartition.new(table_name, 1) + end + + def next_partition + SingleNumericListPartition.new(table_name, active_partition.value + 1) + end + + def extra_partitions + possibly_extra = current_partitions[0...-1] # Never consider the most recent partition + + possibly_extra.take_while { |p| detach_partition_if.call(p.value) } + end + + def after_adding_partitions + active_value = active_partition.value + model.connection.change_column_default(model.table_name, partitioning_key, active_value) + end + + def active_partition + # The current partitions list is sorted, so the last partition has the highest value + # This is the only partition that receives inserts. + current_partitions.last + end + + def no_partitions_exist? + current_partitions.empty? + end + + private + + def ensure_partitioning_column_ignored! + unless model.ignored_columns.include?(partitioning_key.to_s) + raise "Add #{partitioning_key} to #{model.name}.ignored_columns to use it with SlidingListStrategy" + end + end + end + end + end +end diff --git a/lib/gitlab/redis/multi_store.rb b/lib/gitlab/redis/multi_store.rb index f930a0040bc..40dee832b69 100644 --- a/lib/gitlab/redis/multi_store.rb +++ b/lib/gitlab/redis/multi_store.rb @@ -194,7 +194,7 @@ module Gitlab def increment_method_missing_count(command_name) @method_missing_counter ||= Gitlab::Metrics.counter(:gitlab_redis_multi_store_method_missing_total, 'Client side Redis MultiStore method missing') - @method_missing_counter.increment(command: command_name, innamece_name: instance_name) + @method_missing_counter.increment(command: command_name, instance_name: instance_name) end def validate_stores! diff --git a/lib/gitlab/redis/sessions.rb b/lib/gitlab/redis/sessions.rb index 3bf1eb6211d..6cdb8bce12d 100644 --- a/lib/gitlab/redis/sessions.rb +++ b/lib/gitlab/redis/sessions.rb @@ -3,9 +3,45 @@ module Gitlab module Redis class Sessions < ::Gitlab::Redis::Wrapper - # The data we store on Sessions used to be stored on SharedState. - def self.config_fallback - SharedState + SESSION_NAMESPACE = 'session:gitlab' + USER_SESSIONS_NAMESPACE = 'session:user:gitlab' + USER_SESSIONS_LOOKUP_NAMESPACE = 'session:lookup:user:gitlab' + IP_SESSIONS_LOOKUP_NAMESPACE = 'session:lookup:ip:gitlab2' + OTP_SESSIONS_NAMESPACE = 'session:otp' + + class << self + # The data we store on Sessions used to be stored on SharedState. + def config_fallback + SharedState + end + + private + + def redis + # Don't use multistore if redis.sessions configuration is not provided + return super if config_fallback? + + primary_store = ::Redis.new(params) + secondary_store = ::Redis.new(config_fallback.params) + + MultiStore.new(primary_store, secondary_store, name) + end + end + + def store(extras = {}) + # Don't use multistore if redis.sessions configuration is not provided + return super if self.class.config_fallback? + + primary_store = create_redis_store(redis_store_options, extras) + secondary_store = create_redis_store(self.class.config_fallback.params, extras) + + MultiStore.new(primary_store, secondary_store, self.class.name) + end + + private + + def create_redis_store(options, extras) + ::Redis::Store.new(options.merge(extras)) end end end diff --git a/lib/gitlab/redis/sessions_store_helper.rb b/lib/gitlab/redis/sessions_store_helper.rb new file mode 100644 index 00000000000..c80442847f1 --- /dev/null +++ b/lib/gitlab/redis/sessions_store_helper.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +module Gitlab + module Redis + module SessionsStoreHelper + extend ActiveSupport::Concern + + module StoreMethods + def redis_store_class + use_redis_session_store? ? Gitlab::Redis::Sessions : Gitlab::Redis::SharedState + end + + private + + def use_redis_session_store? + Gitlab::Utils.to_boolean(ENV['GITLAB_USE_REDIS_SESSIONS_STORE'], default: true) + end + end + + include StoreMethods + + included do + extend StoreMethods + end + end + end +end diff --git a/lib/gitlab/redis/shared_state.rb b/lib/gitlab/redis/shared_state.rb index 1250eabb041..fb3a143121b 100644 --- a/lib/gitlab/redis/shared_state.rb +++ b/lib/gitlab/redis/shared_state.rb @@ -3,10 +3,6 @@ module Gitlab module Redis class SharedState < ::Gitlab::Redis::Wrapper - SESSION_NAMESPACE = 'session:gitlab' - USER_SESSIONS_NAMESPACE = 'session:user:gitlab' - USER_SESSIONS_LOOKUP_NAMESPACE = 'session:lookup:user:gitlab' - IP_SESSIONS_LOOKUP_NAMESPACE = 'session:lookup:ip:gitlab2' end end end diff --git a/lib/gitlab/redis/wrapper.rb b/lib/gitlab/redis/wrapper.rb index 985c8dc619c..75dbccb965d 100644 --- a/lib/gitlab/redis/wrapper.rb +++ b/lib/gitlab/redis/wrapper.rb @@ -28,7 +28,7 @@ module Gitlab end def pool - @pool ||= ConnectionPool.new(size: pool_size) { ::Redis.new(params) } + @pool ||= ConnectionPool.new(size: pool_size) { redis } end def pool_size @@ -67,6 +67,10 @@ module Gitlab File.expand_path('../../..', __dir__) end + def config_fallback? + config_file_name == config_fallback&.config_file_name + end + def config_file_name [ # Instance specific config sources: @@ -100,6 +104,12 @@ module Gitlab "::Gitlab::Instrumentation::Redis::#{store_name}".constantize end + + private + + def redis + ::Redis.new(params) + end end def initialize(rails_env = nil) diff --git a/lib/gitlab/sidekiq_config.rb b/lib/gitlab/sidekiq_config.rb index 07ddac209f8..3eef41a2ca2 100644 --- a/lib/gitlab/sidekiq_config.rb +++ b/lib/gitlab/sidekiq_config.rb @@ -8,6 +8,7 @@ module Gitlab EE_QUEUE_CONFIG_PATH = 'ee/app/workers/all_queues.yml' JH_QUEUE_CONFIG_PATH = 'jh/app/workers/all_queues.yml' SIDEKIQ_QUEUES_PATH = 'config/sidekiq_queues.yml' + JH_SIDEKIQ_QUEUES_PATH = 'jh/config/sidekiq_queues.yml' QUEUE_CONFIG_PATHS = [ FOSS_QUEUE_CONFIG_PATH, @@ -100,18 +101,24 @@ module Gitlab def queues_for_sidekiq_queues_yml namespaces_with_equal_weights = workers + .reject { |worker| worker.jh? } .group_by(&:queue_namespace) .map(&:last) .select { |workers| workers.map(&:get_weight).uniq.count == 1 } .map(&:first) namespaces = namespaces_with_equal_weights.map(&:queue_namespace).to_set - remaining_queues = workers.reject { |worker| namespaces.include?(worker.queue_namespace) } + remaining_queues = workers.reject { |worker| worker.jh? }.reject { |worker| namespaces.include?(worker.queue_namespace) } (namespaces_with_equal_weights.map(&:namespace_and_weight) + remaining_queues.map(&:queue_and_weight)).sort end + # Override in JH repo + def jh_queues_for_sidekiq_queues_yml + [] + end + # YAML.load_file is OK here as we control the file contents def sidekiq_queues_yml_outdated? config_queues = YAML.load_file(SIDEKIQ_QUEUES_PATH)[:queues] @@ -154,3 +161,5 @@ module Gitlab end end end + +Gitlab::SidekiqConfig.prepend_mod |