diff options
Diffstat (limited to 'lib/gitlab/git')
| -rw-r--r-- | lib/gitlab/git/repository.rb | 4 | ||||
| -rw-r--r-- | lib/gitlab/git/storage.rb | 25 | ||||
| -rw-r--r-- | lib/gitlab/git/storage/checker.rb | 120 | ||||
| -rw-r--r-- | lib/gitlab/git/storage/circuit_breaker.rb | 78 | ||||
| -rw-r--r-- | lib/gitlab/git/storage/circuit_breaker_settings.rb | 37 | ||||
| -rw-r--r-- | lib/gitlab/git/storage/failure_info.rb | 39 | ||||
| -rw-r--r-- | lib/gitlab/git/storage/forked_storage_check.rb | 65 | ||||
| -rw-r--r-- | lib/gitlab/git/storage/health.rb | 92 | ||||
| -rw-r--r-- | lib/gitlab/git/storage/null_circuit_breaker.rb | 50 |
9 files changed, 0 insertions, 510 deletions
diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index 7732049b69b..4218e812146 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -96,10 +96,6 @@ module Gitlab raise Gitlab::Git::CommandError.new(e.message) end - def circuit_breaker - @circuit_breaker ||= Gitlab::Git::Storage::CircuitBreaker.for_storage(storage) - end - def exists? gitaly_repository_client.exists? end diff --git a/lib/gitlab/git/storage.rb b/lib/gitlab/git/storage.rb deleted file mode 100644 index 5933312b0b5..00000000000 --- a/lib/gitlab/git/storage.rb +++ /dev/null @@ -1,25 +0,0 @@ -module Gitlab - module Git - module Storage - class Inaccessible < StandardError - attr_reader :retry_after - - def initialize(message = nil, retry_after = nil) - super(message) - @retry_after = retry_after - end - end - - CircuitOpen = Class.new(Inaccessible) - Misconfiguration = Class.new(Inaccessible) - Failing = Class.new(Inaccessible) - - REDIS_KEY_PREFIX = 'storage_accessible:'.freeze - REDIS_KNOWN_KEYS = "#{REDIS_KEY_PREFIX}known_keys_set".freeze - - def self.redis - Gitlab::Redis::SharedState - end - end - end -end diff --git a/lib/gitlab/git/storage/checker.rb b/lib/gitlab/git/storage/checker.rb deleted file mode 100644 index 391f0d70583..00000000000 --- a/lib/gitlab/git/storage/checker.rb +++ /dev/null @@ -1,120 +0,0 @@ -module Gitlab - module Git - module Storage - class Checker - include CircuitBreakerSettings - - attr_reader :storage_path, :storage, :hostname, :logger - METRICS_MUTEX = Mutex.new - STORAGE_TIMING_BUCKETS = [0.1, 0.15, 0.25, 0.33, 0.5, 1, 1.5, 2.5, 5, 10, 15].freeze - - def self.check_all(logger = Rails.logger) - threads = Gitlab.config.repositories.storages.keys.map do |storage_name| - Thread.new do - Thread.current[:result] = new(storage_name, logger).check_with_lease - end - end - - threads.map do |thread| - thread.join - thread[:result] - end - end - - def self.check_histogram - @check_histogram ||= - METRICS_MUTEX.synchronize do - @check_histogram || Gitlab::Metrics.histogram(:circuitbreaker_storage_check_duration_seconds, - 'Storage check time in seconds', - {}, - STORAGE_TIMING_BUCKETS - ) - end - end - - def initialize(storage, logger = Rails.logger) - @storage = storage - config = Gitlab.config.repositories.storages[@storage] - @storage_path = Gitlab::GitalyClient::StorageSettings.allow_disk_access { config.legacy_disk_path } - @logger = logger - - @hostname = Gitlab::Environment.hostname - end - - def check_with_lease - lease_key = "storage_check:#{cache_key}" - lease = Gitlab::ExclusiveLease.new(lease_key, timeout: storage_timeout) - result = { storage: storage, success: nil } - - if uuid = lease.try_obtain - result[:success] = check - - Gitlab::ExclusiveLease.cancel(lease_key, uuid) - else - logger.warn("#{hostname}: #{storage}: Skipping check, previous check still running") - end - - result - end - - def check - if perform_access_check - track_storage_accessible - true - else - track_storage_inaccessible - logger.error("#{hostname}: #{storage}: Not accessible.") - false - end - end - - private - - def perform_access_check - start_time = Gitlab::Metrics::System.monotonic_time - - Gitlab::Git::Storage::ForkedStorageCheck.storage_available?(storage_path, storage_timeout, access_retries) - ensure - execution_time = Gitlab::Metrics::System.monotonic_time - start_time - self.class.check_histogram.observe({ storage: storage }, execution_time) - end - - def track_storage_inaccessible - first_failure = current_failure_info.first_failure || Time.now - last_failure = Time.now - - Gitlab::Git::Storage.redis.with do |redis| - redis.pipelined do - redis.hset(cache_key, :first_failure, first_failure.to_i) - redis.hset(cache_key, :last_failure, last_failure.to_i) - redis.hincrby(cache_key, :failure_count, 1) - redis.expire(cache_key, failure_reset_time) - maintain_known_keys(redis) - end - end - end - - def track_storage_accessible - Gitlab::Git::Storage.redis.with do |redis| - redis.pipelined do - redis.hset(cache_key, :first_failure, nil) - redis.hset(cache_key, :last_failure, nil) - redis.hset(cache_key, :failure_count, 0) - maintain_known_keys(redis) - end - end - end - - def maintain_known_keys(redis) - expire_time = Time.now.to_i + failure_reset_time - redis.zadd(Gitlab::Git::Storage::REDIS_KNOWN_KEYS, expire_time, cache_key) - redis.zremrangebyscore(Gitlab::Git::Storage::REDIS_KNOWN_KEYS, '-inf', Time.now.to_i) - end - - def current_failure_info - FailureInfo.load(cache_key) - end - end - end - end -end diff --git a/lib/gitlab/git/storage/circuit_breaker.rb b/lib/gitlab/git/storage/circuit_breaker.rb deleted file mode 100644 index fcee9ae566c..00000000000 --- a/lib/gitlab/git/storage/circuit_breaker.rb +++ /dev/null @@ -1,78 +0,0 @@ -module Gitlab - module Git - module Storage - class CircuitBreaker - include CircuitBreakerSettings - - attr_reader :storage, - :hostname - - delegate :last_failure, :failure_count, :no_failures?, - to: :failure_info - - def self.for_storage(storage) - cached_circuitbreakers = Gitlab::SafeRequestStore.fetch(:circuitbreaker_cache) do - Hash.new do |hash, storage_name| - hash[storage_name] = build(storage_name) - end - end - - cached_circuitbreakers[storage] - end - - def self.build(storage, hostname = Gitlab::Environment.hostname) - config = Gitlab.config.repositories.storages[storage] - Gitlab::GitalyClient::StorageSettings.allow_disk_access do - if !config.present? - NullCircuitBreaker.new(storage, hostname, error: Misconfiguration.new("Storage '#{storage}' is not configured")) - elsif !config.legacy_disk_path.present? - NullCircuitBreaker.new(storage, hostname, error: Misconfiguration.new("Path for storage '#{storage}' is not configured")) - else - new(storage, hostname) - end - end - end - - def initialize(storage, hostname) - @storage = storage - @hostname = hostname - end - - def perform - return yield unless enabled? - - check_storage_accessible! - - yield - end - - def circuit_broken? - return false if no_failures? - - failure_count > failure_count_threshold - end - - private - - # The circuitbreaker can be enabled for the entire fleet using a Feature - # flag. - # - # Enabling it for a single host can be done setting the - # `GIT_STORAGE_CIRCUIT_BREAKER` environment variable. - def enabled? - ENV['GIT_STORAGE_CIRCUIT_BREAKER'].present? || Feature.enabled?('git_storage_circuit_breaker') - end - - def failure_info - @failure_info ||= FailureInfo.load(cache_key) - end - - def check_storage_accessible! - if circuit_broken? - raise Gitlab::Git::Storage::CircuitOpen.new("Circuit for #{storage} is broken", failure_reset_time) - end - end - end - end - end -end diff --git a/lib/gitlab/git/storage/circuit_breaker_settings.rb b/lib/gitlab/git/storage/circuit_breaker_settings.rb deleted file mode 100644 index c9e225f187d..00000000000 --- a/lib/gitlab/git/storage/circuit_breaker_settings.rb +++ /dev/null @@ -1,37 +0,0 @@ -module Gitlab - module Git - module Storage - module CircuitBreakerSettings - def failure_count_threshold - application_settings.circuitbreaker_failure_count_threshold - end - - def failure_reset_time - application_settings.circuitbreaker_failure_reset_time - end - - def storage_timeout - application_settings.circuitbreaker_storage_timeout - end - - def access_retries - application_settings.circuitbreaker_access_retries - end - - def check_interval - application_settings.circuitbreaker_check_interval - end - - def cache_key - @cache_key ||= "#{Gitlab::Git::Storage::REDIS_KEY_PREFIX}#{storage}:#{hostname}" - end - - private - - def application_settings - Gitlab::CurrentSettings.current_application_settings - end - end - end - end -end diff --git a/lib/gitlab/git/storage/failure_info.rb b/lib/gitlab/git/storage/failure_info.rb deleted file mode 100644 index 1d28a850049..00000000000 --- a/lib/gitlab/git/storage/failure_info.rb +++ /dev/null @@ -1,39 +0,0 @@ -module Gitlab - module Git - module Storage - class FailureInfo - attr_accessor :first_failure, :last_failure, :failure_count - - def self.reset_all! - Gitlab::Git::Storage.redis.with do |redis| - all_storage_keys = redis.zrange(Gitlab::Git::Storage::REDIS_KNOWN_KEYS, 0, -1) - redis.del(*all_storage_keys) unless all_storage_keys.empty? - end - - Gitlab::SafeRequestStore.delete(:circuitbreaker_cache) - end - - def self.load(cache_key) - first_failure, last_failure, failure_count = Gitlab::Git::Storage.redis.with do |redis| - redis.hmget(cache_key, :first_failure, :last_failure, :failure_count) - end - - last_failure = Time.at(last_failure.to_i) if last_failure.present? - first_failure = Time.at(first_failure.to_i) if first_failure.present? - - new(first_failure, last_failure, failure_count.to_i) - end - - def initialize(first_failure, last_failure, failure_count) - @first_failure = first_failure - @last_failure = last_failure - @failure_count = failure_count - end - - def no_failures? - first_failure.blank? && last_failure.blank? && failure_count == 0 - end - end - end - end -end diff --git a/lib/gitlab/git/storage/forked_storage_check.rb b/lib/gitlab/git/storage/forked_storage_check.rb deleted file mode 100644 index 0a4e557b59b..00000000000 --- a/lib/gitlab/git/storage/forked_storage_check.rb +++ /dev/null @@ -1,65 +0,0 @@ -module Gitlab - module Git - module Storage - module ForkedStorageCheck - extend self - - def storage_available?(path, timeout_seconds = 5, retries = 1) - partial_timeout = timeout_seconds / retries - status = timeout_check(path, partial_timeout) - - # If the status check did not succeed the first time, we retry a few - # more times to avoid one-off failures - current_attempts = 1 - while current_attempts < retries && !status.success? - status = timeout_check(path, partial_timeout) - current_attempts += 1 - end - - status.success? - end - - def timeout_check(path, timeout_seconds) - filesystem_check_pid = check_filesystem_in_process(path) - - deadline = timeout_seconds.seconds.from_now.utc - wait_time = 0.01 - status = nil - - while status.nil? - - if deadline > Time.now.utc - sleep(wait_time) - _pid, status = Process.wait2(filesystem_check_pid, Process::WNOHANG) - else - Process.kill('KILL', filesystem_check_pid) - # Blocking wait, so we are sure the process is gone before continuing - _pid, status = Process.wait2(filesystem_check_pid) - end - end - - status - end - - # This will spawn a new 2 processes to do the check: - # The outer child (waiter) will spawn another child process (stater). - # - # The stater is the process is performing the actual filesystem check - # the check might hang if the filesystem is acting up. - # In this case we will send a `KILL` to the waiter, which will still - # be responsive while the stater is hanging. - def check_filesystem_in_process(path) - spawn('ruby', '-e', ruby_check, path, [:out, :err] => '/dev/null') - end - - def ruby_check - <<~RUBY_FILESYSTEM_CHECK - inner_pid = fork { File.stat(ARGV.first) } - Process.waitpid(inner_pid) - exit $?.exitstatus - RUBY_FILESYSTEM_CHECK - end - end - end - end -end diff --git a/lib/gitlab/git/storage/health.rb b/lib/gitlab/git/storage/health.rb deleted file mode 100644 index 8e14acb4ccb..00000000000 --- a/lib/gitlab/git/storage/health.rb +++ /dev/null @@ -1,92 +0,0 @@ -module Gitlab - module Git - module Storage - class Health - attr_reader :storage_name, :info - - def self.prefix_for_storage(storage_name) - "#{Gitlab::Git::Storage::REDIS_KEY_PREFIX}#{storage_name}:" - end - - def self.for_all_storages - storage_names = Gitlab.config.repositories.storages.keys - results_per_storage = nil - - Gitlab::Git::Storage.redis.with do |redis| - keys_per_storage = all_keys_for_storages(storage_names, redis) - results_per_storage = load_for_keys(keys_per_storage, redis) - end - - results_per_storage.map do |name, info| - info.each { |i| i[:failure_count] = i[:failure_count].value.to_i } - new(name, info) - end - end - - private_class_method def self.all_keys_for_storages(storage_names, redis) - keys_per_storage = {} - all_keys = redis.zrange(Gitlab::Git::Storage::REDIS_KNOWN_KEYS, 0, -1) - - storage_names.each do |storage_name| - prefix = prefix_for_storage(storage_name) - - keys_per_storage[storage_name] = all_keys.select { |key| key.starts_with?(prefix) } - end - - keys_per_storage - end - - private_class_method def self.load_for_keys(keys_per_storage, redis) - info_for_keys = {} - - redis.pipelined do - keys_per_storage.each do |storage_name, keys_future| - info_for_storage = keys_future.map do |key| - { name: key, failure_count: redis.hget(key, :failure_count) } - end - - info_for_keys[storage_name] = info_for_storage - end - end - - info_for_keys - end - - def self.for_failing_storages - for_all_storages.select(&:failing?) - end - - def initialize(storage_name, info) - @storage_name = storage_name - @info = info - end - - def failing_info - @failing_info ||= info.select { |info_for_host| info_for_host[:failure_count] > 0 } - end - - def failing? - failing_info.any? - end - - def failing_on_hosts - @failing_on_hosts ||= failing_info.map do |info_for_host| - info_for_host[:name].split(':').last - end - end - - def failing_circuit_breakers - @failing_circuit_breakers ||= failing_on_hosts.map do |hostname| - CircuitBreaker.build(storage_name, hostname) - end - end - - # rubocop: disable CodeReuse/ActiveRecord - def total_failures - @total_failures ||= failing_info.sum { |info_for_host| info_for_host[:failure_count] } - end - # rubocop: enable CodeReuse/ActiveRecord - end - end - end -end diff --git a/lib/gitlab/git/storage/null_circuit_breaker.rb b/lib/gitlab/git/storage/null_circuit_breaker.rb deleted file mode 100644 index 261c936c689..00000000000 --- a/lib/gitlab/git/storage/null_circuit_breaker.rb +++ /dev/null @@ -1,50 +0,0 @@ -module Gitlab - module Git - module Storage - class NullCircuitBreaker - include CircuitBreakerSettings - - # These will have actual values - attr_reader :storage, - :hostname - - # These will always have nil values - attr_reader :storage_path - - delegate :last_failure, :failure_count, :no_failures?, - to: :failure_info - - def initialize(storage, hostname, error: nil) - @storage = storage - @hostname = hostname - @error = error - end - - def perform - @error ? raise(@error) : yield - end - - def circuit_broken? - !!@error - end - - def backing_off? - false - end - - def failure_info - @failure_info ||= - if circuit_broken? - Gitlab::Git::Storage::FailureInfo.new(Time.now, - Time.now, - failure_count_threshold) - else - Gitlab::Git::Storage::FailureInfo.new(nil, - nil, - 0) - end - end - end - end - end -end |