diff options
Diffstat (limited to 'lib/gitlab/git/storage')
| -rw-r--r-- | lib/gitlab/git/storage/circuit_breaker.rb | 55 | ||||
| -rw-r--r-- | lib/gitlab/git/storage/circuit_breaker_settings.rb | 37 | ||||
| -rw-r--r-- | lib/gitlab/git/storage/forked_storage_check.rb | 13 | ||||
| -rw-r--r-- | lib/gitlab/git/storage/health.rb | 20 | ||||
| -rw-r--r-- | lib/gitlab/git/storage/null_circuit_breaker.rb | 13 |
5 files changed, 103 insertions, 35 deletions
diff --git a/lib/gitlab/git/storage/circuit_breaker.rb b/lib/gitlab/git/storage/circuit_breaker.rb index 1eaa2d83fb6..be7598ef011 100644 --- a/lib/gitlab/git/storage/circuit_breaker.rb +++ b/lib/gitlab/git/storage/circuit_breaker.rb @@ -2,15 +2,13 @@ module Gitlab module Git module Storage class CircuitBreaker + include CircuitBreakerSettings + FailureInfo = Struct.new(:last_failure, :failure_count) attr_reader :storage, :hostname, - :storage_path, - :failure_count_threshold, - :failure_wait_time, - :failure_reset_time, - :storage_timeout + :storage_path delegate :last_failure, :failure_count, to: :failure_info @@ -18,7 +16,7 @@ module Gitlab pattern = "#{Gitlab::Git::Storage::REDIS_KEY_PREFIX}*" Gitlab::Git::Storage.redis.with do |redis| - all_storage_keys = redis.keys(pattern) + all_storage_keys = redis.scan_each(match: pattern).to_a redis.del(*all_storage_keys) unless all_storage_keys.empty? end @@ -53,14 +51,10 @@ module Gitlab config = Gitlab.config.repositories.storages[@storage] @storage_path = config['path'] - @failure_count_threshold = config['failure_count_threshold'] - @failure_wait_time = config['failure_wait_time'] - @failure_reset_time = config['failure_reset_time'] - @storage_timeout = config['storage_timeout'] end def perform - return yield unless Feature.enabled?('git_storage_circuit_breaker') + return yield unless enabled? check_storage_accessible! @@ -70,10 +64,27 @@ module Gitlab def circuit_broken? return false if no_failures? + failure_count > failure_count_threshold + end + + def backing_off? + return false if no_failures? + recent_failure = last_failure > failure_wait_time.seconds.ago - too_many_failures = failure_count > failure_count_threshold + too_many_failures = failure_count > backoff_threshold - recent_failure || too_many_failures + recent_failure && too_many_failures + end + + private + + # The circuitbreaker can be enabled for the entire fleet using a Feature + # flag. + # + # Enabling it for a single host can be done setting the + # `GIT_STORAGE_CIRCUIT_BREAKER` environment variable. + def enabled? + ENV['GIT_STORAGE_CIRCUIT_BREAKER'].present? || Feature.enabled?('git_storage_circuit_breaker') end def failure_info @@ -89,7 +100,7 @@ module Gitlab return @storage_available if @storage_available if @storage_available = Gitlab::Git::Storage::ForkedStorageCheck - .storage_available?(storage_path, storage_timeout) + .storage_available?(storage_path, storage_timeout, access_retries) track_storage_accessible else track_storage_inaccessible @@ -100,7 +111,11 @@ module Gitlab def check_storage_accessible! if circuit_broken? - raise Gitlab::Git::Storage::CircuitOpen.new("Circuit for #{storage} is broken", failure_wait_time) + raise Gitlab::Git::Storage::CircuitOpen.new("Circuit for #{storage} is broken", failure_reset_time) + end + + if backing_off? + raise Gitlab::Git::Storage::Failing.new("Backing off access to #{storage}", failure_wait_time) end unless storage_available? @@ -137,12 +152,6 @@ module Gitlab end end - def cache_key - @cache_key ||= "#{Gitlab::Git::Storage::REDIS_KEY_PREFIX}#{storage}:#{hostname}" - end - - private - def get_failure_info last_failure, failure_count = Gitlab::Git::Storage.redis.with do |redis| redis.hmget(cache_key, :last_failure, :failure_count) @@ -152,6 +161,10 @@ module Gitlab FailureInfo.new(last_failure, failure_count.to_i) end + + def cache_key + @cache_key ||= "#{Gitlab::Git::Storage::REDIS_KEY_PREFIX}#{storage}:#{hostname}" + end end end end diff --git a/lib/gitlab/git/storage/circuit_breaker_settings.rb b/lib/gitlab/git/storage/circuit_breaker_settings.rb new file mode 100644 index 00000000000..257fe8cd8f0 --- /dev/null +++ b/lib/gitlab/git/storage/circuit_breaker_settings.rb @@ -0,0 +1,37 @@ +module Gitlab + module Git + module Storage + module CircuitBreakerSettings + def failure_count_threshold + application_settings.circuitbreaker_failure_count_threshold + end + + def failure_wait_time + application_settings.circuitbreaker_failure_wait_time + end + + def failure_reset_time + application_settings.circuitbreaker_failure_reset_time + end + + def storage_timeout + application_settings.circuitbreaker_storage_timeout + end + + def access_retries + application_settings.circuitbreaker_access_retries + end + + def backoff_threshold + application_settings.circuitbreaker_backoff_threshold + end + + private + + def application_settings + Gitlab::CurrentSettings.current_application_settings + end + end + end + end +end diff --git a/lib/gitlab/git/storage/forked_storage_check.rb b/lib/gitlab/git/storage/forked_storage_check.rb index 91d8241f17b..1307f400700 100644 --- a/lib/gitlab/git/storage/forked_storage_check.rb +++ b/lib/gitlab/git/storage/forked_storage_check.rb @@ -4,8 +4,17 @@ module Gitlab module ForkedStorageCheck extend self - def storage_available?(path, timeout_seconds = 5) - status = timeout_check(path, timeout_seconds) + def storage_available?(path, timeout_seconds = 5, retries = 1) + partial_timeout = timeout_seconds / retries + status = timeout_check(path, partial_timeout) + + # If the status check did not succeed the first time, we retry a few + # more times to avoid one-off failures + current_attempts = 1 + while current_attempts < retries && !status.success? + status = timeout_check(path, partial_timeout) + current_attempts += 1 + end status.success? end diff --git a/lib/gitlab/git/storage/health.rb b/lib/gitlab/git/storage/health.rb index 1564e94b7f7..7049772fe3b 100644 --- a/lib/gitlab/git/storage/health.rb +++ b/lib/gitlab/git/storage/health.rb @@ -23,26 +23,36 @@ module Gitlab end end - def self.all_keys_for_storages(storage_names, redis) + private_class_method def self.all_keys_for_storages(storage_names, redis) keys_per_storage = {} redis.pipelined do storage_names.each do |storage_name| pattern = pattern_for_storage(storage_name) + matched_keys = redis.scan_each(match: pattern) - keys_per_storage[storage_name] = redis.keys(pattern) + keys_per_storage[storage_name] = matched_keys end end - keys_per_storage + # We need to make sure each lazy-loaded `Enumerator` for matched keys + # is loaded into an array. + # + # Otherwise it would be loaded in the second `Redis#pipelined` block + # within `.load_for_keys`. In this pipelined call, the active + # Redis-client changes again, so the values would not be available + # until the end of that pipelined-block. + keys_per_storage.each do |storage_name, key_future| + keys_per_storage[storage_name] = key_future.to_a + end end - def self.load_for_keys(keys_per_storage, redis) + private_class_method def self.load_for_keys(keys_per_storage, redis) info_for_keys = {} redis.pipelined do keys_per_storage.each do |storage_name, keys_future| - info_for_storage = keys_future.value.map do |key| + info_for_storage = keys_future.map do |key| { name: key, failure_count: redis.hget(key, :failure_count) } end diff --git a/lib/gitlab/git/storage/null_circuit_breaker.rb b/lib/gitlab/git/storage/null_circuit_breaker.rb index 297c043d054..a12d52d295f 100644 --- a/lib/gitlab/git/storage/null_circuit_breaker.rb +++ b/lib/gitlab/git/storage/null_circuit_breaker.rb @@ -2,15 +2,14 @@ module Gitlab module Git module Storage class NullCircuitBreaker + include CircuitBreakerSettings + # These will have actual values attr_reader :storage, :hostname # These will always have nil values - attr_reader :storage_path, - :failure_wait_time, - :failure_reset_time, - :storage_timeout + attr_reader :storage_path def initialize(storage, hostname, error: nil) @storage = storage @@ -26,8 +25,8 @@ module Gitlab !!@error end - def failure_count_threshold - 1 + def backing_off? + false end def last_failure @@ -35,7 +34,7 @@ module Gitlab end def failure_count - circuit_broken? ? 1 : 0 + circuit_broken? ? failure_count_threshold : 0 end def failure_info |