Add a Circuitbreaker for storage paths

4 files changed, 40 insertions, 1 deletions

diff --git a/config/gitlab.yml.example b/config/gitlab.yml.example
index 73a68c6da1b..45ab4e1a851 100644
--- a/config/gitlab.yml.example
+++ b/config/gitlab.yml.example
@@ -506,6 +506,11 @@ production: &base
         path: /home/git/repositories/
         gitaly_address: unix:/home/git/gitlab/tmp/sockets/private/gitaly.socket # TCP connections are supported too (e.g. tcp://host:port)
         # gitaly_token: 'special token' # Optional: override global gitaly.token for this storage.
+        failure_count_threshold: 10 # number of failures before stopping attempts
+        failure_wait_time: 30 # Seconds after an access failure before allowing access again
+        failure_reset_time: 1800 # Time in seconds to expire failures
+        storage_timeout: 5 # Time in seconds to wait before aborting a storage access attempt
+
 
   ## Backup settings
   backup:
@@ -638,6 +643,10 @@ test:
       default:
         path: tmp/tests/repositories/
         gitaly_address: unix:tmp/tests/gitaly/gitaly.socket
+      broken:
+        path: tmp/tests/non-existent-repositories
+        gitaly_address: unix:tmp/tests/gitaly/gitaly.socket
+
   gitaly:
     enabled: true
     token: secret
diff --git a/config/initializers/1_settings.rb b/config/initializers/1_settings.rb
index 63f4c8c9e0a..017537f30be 100644
--- a/config/initializers/1_settings.rb
+++ b/config/initializers/1_settings.rb
@@ -222,6 +222,7 @@ Settings.gitlab['default_branch_protection'] ||= 2
 Settings.gitlab['default_can_create_group'] = true if Settings.gitlab['default_can_create_group'].nil?
 Settings.gitlab['host']       ||= ENV['GITLAB_HOST'] || 'localhost'
 Settings.gitlab['ssh_host']   ||= Settings.gitlab.host
+Settings.gitlab['hostname']   ||= ENV['HOSTNAME'] || Socket.gethostname
 Settings.gitlab['https']        = false if Settings.gitlab['https'].nil?
 Settings.gitlab['port']       ||= ENV['GITLAB_PORT'] || (Settings.gitlab.https ? 443 : 80)
 Settings.gitlab['relative_url_root'] ||= ENV['RAILS_RELATIVE_URL_ROOT'] || ''
@@ -433,6 +434,17 @@ end
 Settings.repositories.storages.values.each do |storage|
   # Expand relative paths
   storage['path'] = Settings.absolute(storage['path'])
+  # Set failure defaults
+  storage['failure_count_threshold'] ||= 10
+  storage['failure_wait_time'] ||= 30
+  storage['failure_reset_time'] ||= 1800
+  storage['storage_timeout'] ||= 5
+  # Set turn strings into numbers
+  storage['failure_count_threshold'] = storage['failure_count_threshold'].to_i
+  storage['failure_wait_time'] = storage['failure_wait_time'].to_i
+  storage['failure_reset_time'] = storage['failure_reset_time'].to_i
+  # We might want to have a timeout shorter than 1 second.
+  storage['storage_timeout'] = storage['storage_timeout'].to_f
 end
 
 #
diff --git a/config/initializers/6_validations.rb b/config/initializers/6_validations.rb
index 9e24f42d284..92ce4dd03cd 100644
--- a/config/initializers/6_validations.rb
+++ b/config/initializers/6_validations.rb
@@ -7,6 +7,13 @@ def find_parent_path(name, path)
   Gitlab.config.repositories.storages.detect do |n, rs|
     name != n && Pathname.new(rs['path']).realpath == parent
   end
+rescue Errno::EIO, Errno::ENOENT => e
+  warning = "WARNING: couldn't verify #{path} (#{name}). "\
+            "If this is an external storage, it might be offline."
+  message = "#{warning}\n#{e.message}"
+  Rails.logger.error("#{message}\n\t" + e.backtrace.join("\n\t"))
+
+  nil
 end
 
 def storage_validation_error(message)
@@ -29,6 +36,15 @@ def validate_storages_config
     if !repository_storage.is_a?(Hash) || repository_storage['path'].nil?
       storage_validation_error("#{name} is not a valid storage, because it has no `path` key. Refer to gitlab.yml.example for an updated example")
     end
+
+    %w(failure_count_threshold failure_wait_time failure_reset_time storage_timeout).each do |setting|
+      # Falling back to the defaults is fine!
+      next if repository_storage[setting].nil?
+
+      unless repository_storage[setting].to_f > 0
+        storage_validation_error("#{setting}, for storage `#{name}` needs to be greater than 0")
+      end
+    end
   end
 end
 
diff --git a/config/routes/admin.rb b/config/routes/admin.rb
index 5427bab93ce..c0748231813 100644
--- a/config/routes/admin.rb
+++ b/config/routes/admin.rb
@@ -67,7 +67,9 @@ namespace :admin do
   end
 
   resource :logs, only: [:show]
-  resource :health_check, controller: 'health_check', only: [:show]
+  resource :health_check, controller: 'health_check', only: [:show] do
+    post :reset_storage_health
+  end
   resource :background_jobs, controller: 'background_jobs', only: [:show]
   resource :system_info, controller: 'system_info', only: [:show]
   resources :requests_profiles, only: [:index, :show], param: :name, constraints: { name: /.+\.html/ }