diff options
Diffstat (limited to 'lib/gitlab/database/load_balancing')
5 files changed, 78 insertions, 3 deletions
diff --git a/lib/gitlab/database/load_balancing/configuration.rb b/lib/gitlab/database/load_balancing/configuration.rb index 59b08fac7e9..50472bd5780 100644 --- a/lib/gitlab/database/load_balancing/configuration.rb +++ b/lib/gitlab/database/load_balancing/configuration.rb @@ -57,7 +57,8 @@ module Gitlab record_type: 'A', interval: 60, disconnect_timeout: 120, - use_tcp: false + use_tcp: false, + max_replica_pools: nil } end diff --git a/lib/gitlab/database/load_balancing/load_balancer.rb b/lib/gitlab/database/load_balancing/load_balancer.rb index 0881025b425..cb3a378ad64 100644 --- a/lib/gitlab/database/load_balancing/load_balancer.rb +++ b/lib/gitlab/database/load_balancing/load_balancer.rb @@ -119,6 +119,13 @@ module Gitlab connection = pool.connection transaction_open = connection.transaction_open? + if attempt && attempt > 1 + ::Gitlab::Database::LoadBalancing::Logger.warn( + event: :read_write_retry, + message: 'A read_write block was retried because of connection error' + ) + end + yield connection rescue StandardError => e # No leaking will happen on the final attempt. Leaks are caused by subsequent retries diff --git a/lib/gitlab/database/load_balancing/service_discovery.rb b/lib/gitlab/database/load_balancing/service_discovery.rb index dfd4892371c..52a9e8798d4 100644 --- a/lib/gitlab/database/load_balancing/service_discovery.rb +++ b/lib/gitlab/database/load_balancing/service_discovery.rb @@ -48,6 +48,7 @@ module Gitlab # forcefully disconnected. # use_tcp - Use TCP instaed of UDP to look up resources # load_balancer - The load balancer instance to use + # rubocop:disable Metrics/ParameterLists def initialize( load_balancer, nameserver:, @@ -56,7 +57,8 @@ module Gitlab record_type: 'A', interval: 60, disconnect_timeout: 120, - use_tcp: false + use_tcp: false, + max_replica_pools: nil ) @nameserver = nameserver @port = port @@ -66,7 +68,9 @@ module Gitlab @disconnect_timeout = disconnect_timeout @use_tcp = use_tcp @load_balancer = load_balancer + @max_replica_pools = max_replica_pools end + # rubocop:enable Metrics/ParameterLists def start Thread.new do @@ -170,6 +174,8 @@ module Gitlab addresses_from_srv_record(response) end + addresses = sampler.sample(addresses) + raise EmptyDnsResponse if addresses.empty? # Addresses are sorted so we can directly compare the old and new @@ -221,6 +227,11 @@ module Gitlab def addresses_from_a_record(resources) resources.map { |r| Address.new(r.address.to_s) } end + + def sampler + @sampler ||= ::Gitlab::Database::LoadBalancing::ServiceDiscovery::Sampler + .new(max_replica_pools: @max_replica_pools) + end end end end diff --git a/lib/gitlab/database/load_balancing/service_discovery/sampler.rb b/lib/gitlab/database/load_balancing/service_discovery/sampler.rb new file mode 100644 index 00000000000..71870214156 --- /dev/null +++ b/lib/gitlab/database/load_balancing/service_discovery/sampler.rb @@ -0,0 +1,56 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module LoadBalancing + class ServiceDiscovery + class Sampler + def initialize(max_replica_pools:, seed: Random.new_seed) + # seed must be set once and consistent + # for every invocation of #sample on + # the same instance of Sampler + @seed = seed + @max_replica_pools = max_replica_pools + end + + def sample(addresses) + return addresses if @max_replica_pools.nil? || addresses.count <= @max_replica_pools + + ::Gitlab::Database::LoadBalancing::Logger.info( + event: :host_list_limit_exceeded, + message: "Host list length exceeds max_replica_pools so random hosts will be chosen.", + max_replica_pools: @max_replica_pools, + total_host_list_length: addresses.count, + randomization_seed: @seed + ) + + # First sort them in case the ordering from DNS server changes + # then randomly order all addresses using consistent seed so + # this process always gives the same set for this instance of + # Sampler + addresses = addresses.sort + addresses = addresses.shuffle(random: Random.new(@seed)) + + # Group by hostname so that we can sample evenly across hosts + addresses_by_host = addresses.group_by(&:hostname) + + selected_addresses = [] + while selected_addresses.count < @max_replica_pools + # Loop over all hostnames grabbing one address at a time to + # evenly distribute across all hostnames + addresses_by_host.each do |host, addresses| + next if addresses.empty? + + selected_addresses << addresses.pop + + break unless selected_addresses.count < @max_replica_pools + end + end + + selected_addresses + end + end + end + end + end +end diff --git a/lib/gitlab/database/load_balancing/sidekiq_server_middleware.rb b/lib/gitlab/database/load_balancing/sidekiq_server_middleware.rb index 3180289ec69..737852d5ccb 100644 --- a/lib/gitlab/database/load_balancing/sidekiq_server_middleware.rb +++ b/lib/gitlab/database/load_balancing/sidekiq_server_middleware.rb @@ -4,7 +4,7 @@ module Gitlab module Database module LoadBalancing class SidekiqServerMiddleware - JobReplicaNotUpToDate = Class.new(StandardError) + JobReplicaNotUpToDate = Class.new(::Gitlab::SidekiqMiddleware::RetryError) MINIMUM_DELAY_INTERVAL_SECONDS = 0.8 |