Add a Circuitbreaker for storage paths

17 files changed, 699 insertions, 4 deletions

diff --git a/spec/controllers/admin/health_check_controller_spec.rb b/spec/controllers/admin/health_check_controller_spec.rb
new file mode 100644
index 00000000000..0b8e0c8a065
--- /dev/null
+++ b/spec/controllers/admin/health_check_controller_spec.rb
@@ -0,0 +1,25 @@
+require 'spec_helper'
+
+describe Admin::HealthCheckController, broken_storage: true do
+  let(:admin) { create(:admin) }
+
+  before do
+    sign_in(admin)
+  end
+
+  describe 'GET show' do
+    it 'loads the git storage health information' do
+      get :show
+
+      expect(assigns[:failing_storage_statuses]).not_to be_nil
+    end
+  end
+
+  describe 'POST reset_storage_health' do
+    it 'resets all storage health information' do
+      expect(Gitlab::Git::Storage::CircuitBreaker).to receive(:reset_all!)
+
+      post :reset_storage_health
+    end
+  end
+end
diff --git a/spec/controllers/application_controller_spec.rb b/spec/controllers/application_controller_spec.rb
index 1641bddea11..331903a5543 100644
--- a/spec/controllers/application_controller_spec.rb
+++ b/spec/controllers/application_controller_spec.rb
@@ -108,6 +108,30 @@ describe ApplicationController do
     end
   end
 
+  describe 'rescue from Gitlab::Git::Storage::Inaccessible' do
+    controller(described_class) do
+      def index
+        raise Gitlab::Git::Storage::Inaccessible.new('broken', 100)
+      end
+    end
+
+    it 'renders a 503 when storage is not available' do
+      sign_in(create(:user))
+
+      get :index
+
+      expect(response.status).to eq(503)
+    end
+
+    it 'renders includes a Retry-After header' do
+      sign_in(create(:user))
+
+      get :index
+
+      expect(response.headers['Retry-After']).to eq(100)
+    end
+  end
+
   describe 'response format' do
     controller(described_class) do
       def index
diff --git a/spec/controllers/projects_controller_spec.rb b/spec/controllers/projects_controller_spec.rb
index 34095ef6250..8ecd8b6ca71 100644
--- a/spec/controllers/projects_controller_spec.rb
+++ b/spec/controllers/projects_controller_spec.rb
@@ -107,6 +107,20 @@ describe ProjectsController do
       end
     end
 
+    context 'when the storage is not available', broken_storage: true do
+      let(:project) { create(:project, :broken_storage) }
+      before do
+        project.add_developer(user)
+        sign_in(user)
+      end
+
+      it 'renders a 503' do
+        get :show, namespace_id: project.namespace, id: project
+
+        expect(response).to have_http_status(503)
+      end
+    end
+
     context "project with empty repo" do
       let(:empty_project) { create(:project_empty_repo, :public) }
 
diff --git a/spec/factories/projects.rb b/spec/factories/projects.rb
index be3f219e8bf..3f8e7030b1c 100644
--- a/spec/factories/projects.rb
+++ b/spec/factories/projects.rb
@@ -54,6 +54,12 @@ FactoryGirl.define do
       avatar { File.open(Rails.root.join('spec/fixtures/dk.png')) }
     end
 
+    trait :broken_storage do
+      after(:create) do |project|
+        project.update_column(:repository_storage, 'broken')
+      end
+    end
+
     # Test repository - https://gitlab.com/gitlab-org/gitlab-test
     trait :repository do
       path { 'gitlabhq' }
diff --git a/spec/features/admin/admin_health_check_spec.rb b/spec/features/admin/admin_health_check_spec.rb
index 106e7370a98..634dfd53f71 100644
--- a/spec/features/admin/admin_health_check_spec.rb
+++ b/spec/features/admin/admin_health_check_spec.rb
@@ -1,6 +1,6 @@
 require 'spec_helper'
 
-feature "Admin Health Check" do
+feature "Admin Health Check", feature: true, broken_storage: true do
   include StubENV
 
   before do
@@ -55,4 +55,26 @@ feature "Admin Health Check" do
       expect(page).to have_content('The server is on fire')
     end
   end
+
+  context 'with repository storage failures' do
+    before do
+      # Track a failure
+      Gitlab::Git::Storage::CircuitBreaker.for_storage('broken').perform { nil } rescue nil
+      visit admin_health_check_path
+    end
+
+    it 'shows storage failure information' do
+      hostname = Gitlab.config.gitlab.hostname
+
+      expect(page).to have_content('broken: failed storage access attempt on host:')
+      expect(page).to have_content("#{hostname}: 1 of 10 failures.")
+    end
+
+    it 'allows resetting storage failures' do
+      click_button 'Reset git storage health information'
+
+      expect(page).to have_content('Git storage health information has been reset')
+      expect(page).not_to have_content('failed storage access attempt')
+    end
+  end
 end
diff --git a/spec/helpers/storage_health_helper_spec.rb b/spec/helpers/storage_health_helper_spec.rb
new file mode 100644
index 00000000000..874498e6338
--- /dev/null
+++ b/spec/helpers/storage_health_helper_spec.rb
@@ -0,0 +1,20 @@
+require 'spec_helper'
+
+describe StorageHealthHelper do
+  describe '#failing_storage_health_message' do
+    let(:health) do
+      Gitlab::Git::Storage::Health.new(
+        "<script>alert('storage name');)</script>",
+        []
+      )
+    end
+
+    it 'escapes storage names' do
+      escaped_storage_name = '&lt;script&gt;alert(&#39;storage name&#39;);)&lt;/script&gt;'
+
+      result = helper.failing_storage_health_message(health)
+
+      expect(result).to include(escaped_storage_name)
+    end
+  end
+end
diff --git a/spec/initializers/6_validations_spec.rb b/spec/initializers/6_validations_spec.rb
index 0877770c167..83283f03940 100644
--- a/spec/initializers/6_validations_spec.rb
+++ b/spec/initializers/6_validations_spec.rb
@@ -23,6 +23,16 @@ describe '6_validations' do
       end
     end
 
+    context 'when one of the settings is incorrect' do
+      before do
+        mock_storages('foo' => { 'path' => 'tmp/tests/paths/a/b/c', 'failure_count_threshold' => 'not a number' })
+      end
+
+      it 'throws an error' do
+        expect { validate_storages_config }.to raise_error(/failure_count_threshold/)
+      end
+    end
+
     context 'with invalid storage names' do
       before do
         mock_storages('name with spaces' => { 'path' => 'tmp/tests/paths/a/b/c' })
@@ -84,6 +94,17 @@ describe '6_validations' do
         expect { validate_storages_paths }.not_to raise_error
       end
     end
+
+    describe 'inaccessible storage' do
+      before do
+        mock_storages('foo' => { 'path' => 'tmp/tests/a/path/that/does/not/exist' })
+      end
+
+      it 'passes through with a warning' do
+        expect(Rails.logger).to receive(:error)
+        expect { validate_storages_paths }.not_to raise_error
+      end
+    end
   end
 
   def mock_storages(storages)
diff --git a/spec/initializers/settings_spec.rb b/spec/initializers/settings_spec.rb
index ebdabcf93f1..e5ec90cb8f9 100644
--- a/spec/initializers/settings_spec.rb
+++ b/spec/initializers/settings_spec.rb
@@ -2,6 +2,17 @@ require 'spec_helper'
 require_relative '../../config/initializers/1_settings'
 
 describe Settings do
+  describe '#repositories' do
+    it 'assigns the default failure attributes' do
+      repository_settings = Gitlab.config.repositories.storages['broken']
+
+      expect(repository_settings['failure_count_threshold']).to eq(10)
+      expect(repository_settings['failure_wait_time']).to eq(30)
+      expect(repository_settings['failure_reset_time']).to eq(1800)
+      expect(repository_settings['storage_timeout']).to eq(5)
+    end
+  end
+
   describe '#host_without_www' do
     context 'URL with protocol' do
       it 'returns the host' do
diff --git a/spec/lib/gitlab/cache/ci/project_pipeline_status_spec.rb b/spec/lib/gitlab/cache/ci/project_pipeline_status_spec.rb
index f43d89d7ccd..16704ff5e77 100644
--- a/spec/lib/gitlab/cache/ci/project_pipeline_status_spec.rb
+++ b/spec/lib/gitlab/cache/ci/project_pipeline_status_spec.rb
@@ -48,8 +48,9 @@ describe Gitlab::Cache::Ci::ProjectPipelineStatus, :clean_gitlab_redis_cache do
           described_class.load_in_batch_for_projects([project_without_status])
         end
 
-        it 'only connects to redis_cache twice' do
-          # Once to load, once to store in the cache
+        it 'only connects to redis twice' do
+          # Stub circuitbreaker so it doesn't count the redis connections in there
+          stub_circuit_breaker(project_without_status)
           expect(Gitlab::Redis::Cache).to receive(:with).exactly(2).and_call_original
 
           described_class.load_in_batch_for_projects([project_without_status])
@@ -301,4 +302,13 @@ describe Gitlab::Cache::Ci::ProjectPipelineStatus, :clean_gitlab_redis_cache do
       end
     end
   end
+
+  def stub_circuit_breaker(project)
+    fake_circuitbreaker = double
+    allow(fake_circuitbreaker).to receive(:perform).and_yield
+    allow(project.repository.raw_repository)
+      .to receive(:circuit_breaker).and_return(fake_circuitbreaker)
+    allow(project.repository)
+      .to receive(:circuit_breaker).and_return(fake_circuitbreaker)
+  end
 end
diff --git a/spec/lib/gitlab/git/repository_spec.rb b/spec/lib/gitlab/git/repository_spec.rb
index 9bfad0c9bdf..07cd6db2200 100644
--- a/spec/lib/gitlab/git/repository_spec.rb
+++ b/spec/lib/gitlab/git/repository_spec.rb
@@ -55,6 +55,20 @@ describe Gitlab::Git::Repository, seed_helper: true do
   end
 
   describe "#rugged" do
+    describe 'when storage is broken', broken_storage: true  do
+      it 'raises a storage exception when storage is not available' do
+        broken_repo = described_class.new('broken', 'a/path.git')
+
+        expect { broken_repo.rugged }.to raise_error(Gitlab::Git::Storage::Inaccessible)
+      end
+    end
+
+    it 'raises a no repository exception when there is no repo' do
+      broken_repo = described_class.new('default', 'a/path.git')
+
+      expect { broken_repo.rugged }.to raise_error(Gitlab::Git::Repository::NoRepository)
+    end
+
     context 'with no Git env stored' do
       before do
         expect(Gitlab::Git::Env).to receive(:all).and_return({})
diff --git a/spec/lib/gitlab/git/storage/circuit_breaker_spec.rb b/spec/lib/gitlab/git/storage/circuit_breaker_spec.rb
new file mode 100644
index 00000000000..479e53230bc
--- /dev/null
+++ b/spec/lib/gitlab/git/storage/circuit_breaker_spec.rb
@@ -0,0 +1,265 @@
+require 'spec_helper'
+
+describe Gitlab::Git::Storage::CircuitBreaker, clean_gitlab_redis_shared_state: true, broken_storage: true do
+  let(:circuit_breaker) { described_class.new('default') }
+  let(:hostname) { Gitlab.config.gitlab.hostname }
+  let(:cache_key) { "storage_accessible:default:#{hostname}" }
+
+  def value_from_redis(name)
+    Gitlab::Git::Storage.redis.with do |redis|
+      redis.hmget(cache_key, name)
+    end.first
+  end
+
+  def set_in_redis(name, value)
+    Gitlab::Git::Storage.redis.with do |redis|
+      redis.hmset(cache_key, name, value)
+    end.first
+  end
+
+  describe '.reset_all!' do
+    it 'clears all entries form redis' do
+      set_in_redis(:failure_count, 10)
+
+      described_class.reset_all!
+
+      key_exists = Gitlab::Git::Storage.redis.with { |redis| redis.exists(cache_key) }
+
+      expect(key_exists).to be_falsey
+    end
+  end
+
+  describe '.for_storage' do
+    it 'only builds a single circuitbreaker per storage' do
+      expect(described_class).to receive(:new).once.and_call_original
+
+      breaker = described_class.for_storage('default')
+
+      expect(breaker).to be_a(described_class)
+      expect(described_class.for_storage('default')).to eq(breaker)
+    end
+  end
+
+  describe '#initialize' do
+    it 'assigns the settings' do
+      expect(circuit_breaker.hostname).to eq(hostname)
+      expect(circuit_breaker.storage).to eq('default')
+      expect(circuit_breaker.storage_path).to eq(TestEnv.repos_path)
+      expect(circuit_breaker.failure_count_threshold).to eq(10)
+      expect(circuit_breaker.failure_wait_time).to eq(30)
+      expect(circuit_breaker.failure_reset_time).to eq(1800)
+      expect(circuit_breaker.storage_timeout).to eq(5)
+    end
+  end
+
+  describe '#perform' do
+    it 'raises an exception with retry time when the circuit is open' do
+      allow(circuit_breaker).to receive(:circuit_broken?).and_return(true)
+
+      expect { |b| circuit_breaker.perform(&b) }
+        .to raise_error(Gitlab::Git::Storage::CircuitOpen)
+    end
+
+    it 'yields the block' do
+      expect { |b| circuit_breaker.perform(&b) }
+        .to yield_control
+    end
+
+    it 'checks if the storage is available' do
+      expect(circuit_breaker).to receive(:check_storage_accessible!)
+
+      circuit_breaker.perform { 'hello world' }
+    end
+
+    it 'returns the value of the block' do
+      result = circuit_breaker.perform { 'return value' }
+
+      expect(result).to eq('return value')
+    end
+
+    it 'raises possible errors' do
+      expect { circuit_breaker.perform { raise Rugged::OSError.new('Broken') } }
+        .to raise_error(Rugged::OSError)
+    end
+
+    context 'with the feature disabled' do
+      it 'returns the block without checking accessibility' do
+        stub_feature_flags(git_storage_circuit_breaker: false)
+
+        expect(circuit_breaker).not_to receive(:circuit_broken?)
+
+        result = circuit_breaker.perform { 'hello' }
+
+        expect(result).to eq('hello')
+      end
+    end
+  end
+
+  describe '#circuit_broken?' do
+    it 'is closed when there is no last failure' do
+      set_in_redis(:last_failure, nil)
+      set_in_redis(:failure_count, 0)
+
+      expect(circuit_breaker.circuit_broken?).to be_falsey
+    end
+
+    it 'is open when there was a recent failure' do
+      Timecop.freeze do
+        set_in_redis(:last_failure, 1.second.ago.to_f)
+        set_in_redis(:failure_count, 1)
+
+        expect(circuit_breaker.circuit_broken?).to be_truthy
+      end
+    end
+
+    it 'is open when there are to many failures' do
+      set_in_redis(:last_failure, 1.day.ago.to_f)
+      set_in_redis(:failure_count, 200)
+
+      expect(circuit_breaker.circuit_broken?).to be_truthy
+    end
+  end
+
+  describe '#check_storage_accessible!' do
+    context 'when the storage is available' do
+      it 'tracks that the storage was accessible an raises the error' do
+        expect(circuit_breaker).to receive(:track_storage_accessible)
+
+        circuit_breaker.check_storage_accessible!
+      end
+    end
+
+    context 'when the storage is not available' do
+      let(:circuit_breaker) { described_class.new('broken') }
+
+      it 'tracks that the storage was unavailable and raises an error with retry time' do
+        expect(circuit_breaker).to receive(:track_storage_inaccessible)
+
+        expect { circuit_breaker.check_storage_accessible! }
+          .to raise_error do |exception|
+          expect(exception).to be_kind_of(Gitlab::Git::Storage::Inaccessible)
+          expect(exception.retry_after).to eq(30)
+        end
+      end
+    end
+  end
+
+  describe '#track_storage_inaccessible' do
+    around(:each) do |example|
+      Timecop.freeze
+
+      example.run
+
+      Timecop.return
+    end
+
+    it 'records the failure time in redis' do
+      circuit_breaker.track_storage_inaccessible
+
+      failure_time = value_from_redis(:last_failure)
+
+      expect(Time.at(failure_time.to_i)).to be_within(1.second).of(Time.now)
+    end
+
+    it 'sets the failure time on the breaker without reloading' do
+      circuit_breaker.track_storage_inaccessible
+
+      expect(circuit_breaker).not_to receive(:get_failure_info)
+      expect(circuit_breaker.last_failure).to eq(Time.now)
+    end
+
+    it 'increments the failure count in redis' do
+      set_in_redis(:failure_count, 10)
+
+      circuit_breaker.track_storage_inaccessible
+
+      expect(value_from_redis(:failure_count).to_i).to be(11)
+    end
+
+    it 'increments the failure count on the breaker without reloading' do
+      set_in_redis(:failure_count, 10)
+
+      circuit_breaker.track_storage_inaccessible
+
+      expect(circuit_breaker).not_to receive(:get_failure_info)
+      expect(circuit_breaker.failure_count).to eq(11)
+    end
+  end
+
+  describe '#track_storage_accessible' do
+    it 'sets the failure count to zero in redis' do
+      set_in_redis(:failure_count, 10)
+
+      circuit_breaker.track_storage_accessible
+
+      expect(value_from_redis(:failure_count).to_i).to be(0)
+    end
+
+    it 'sets the failure count to zero on the breaker without reloading' do
+      set_in_redis(:failure_count, 10)
+
+      circuit_breaker.track_storage_accessible
+
+      expect(circuit_breaker).not_to receive(:get_failure_info)
+      expect(circuit_breaker.failure_count).to eq(0)
+    end
+
+    it 'removes the last failure time from redis' do
+      set_in_redis(:last_failure, Time.now.to_i)
+
+      circuit_breaker.track_storage_accessible
+
+      expect(circuit_breaker).not_to receive(:get_failure_info)
+      expect(circuit_breaker.last_failure).to be_nil
+    end
+
+    it 'removes the last failure time from the breaker without reloading' do
+      set_in_redis(:last_failure, Time.now.to_i)
+
+      circuit_breaker.track_storage_accessible
+
+      expect(value_from_redis(:last_failure)).to be_empty
+    end
+
+    it 'wont connect to redis when there are no failures' do
+      expect(Gitlab::Git::Storage.redis).to receive(:with).once
+                                              .and_call_original
+      expect(circuit_breaker).to receive(:track_storage_accessible)
+                           .and_call_original
+
+      circuit_breaker.track_storage_accessible
+    end
+  end
+
+  describe '#no_failures?' do
+    it 'is false when a failure was tracked' do
+      set_in_redis(:last_failure, Time.now.to_i)
+      set_in_redis(:failure_count, 1)
+
+      expect(circuit_breaker.no_failures?).to be_falsey
+    end
+  end
+
+  describe '#last_failure' do
+    it 'returns the last failure time' do
+      time = Time.parse("2017-05-26 17:52:30")
+      set_in_redis(:last_failure, time.to_i)
+
+      expect(circuit_breaker.last_failure).to eq(time)
+    end
+  end
+
+  describe '#failure_count' do
+    it 'returns the failure count' do
+      set_in_redis(:failure_count, 7)
+
+      expect(circuit_breaker.failure_count).to eq(7)
+    end
+  end
+
+  describe '#cache_key' do
+    it 'includes storage and host' do
+      expect(circuit_breaker.cache_key).to eq(cache_key)
+    end
+  end
+end
diff --git a/spec/lib/gitlab/git/storage/forked_storage_check_spec.rb b/spec/lib/gitlab/git/storage/forked_storage_check_spec.rb
new file mode 100644
index 00000000000..a9e048b316d
--- /dev/null
+++ b/spec/lib/gitlab/git/storage/forked_storage_check_spec.rb
@@ -0,0 +1,27 @@
+require 'spec_helper'
+
+describe Gitlab::Git::Storage::ForkedStorageCheck, skip_database_cleaner: true do
+  let(:existing_path) do
+    existing_path = TestEnv.repos_path
+    FileUtils.mkdir_p(existing_path)
+    existing_path
+  end
+
+  describe '.storage_accessible?' do
+    it 'detects when a storage is not available' do
+      expect(described_class.storage_available?('/non/existant/path')).to be_falsey
+    end
+
+    it 'detects when a storage is available' do
+      expect(described_class.storage_available?(existing_path)).to be_truthy
+    end
+
+    it 'returns false when the check takes to long' do
+      allow(described_class).to receive(:check_filesystem_in_fork) do
+        fork { sleep 10 }
+      end
+
+      expect(described_class.storage_available?(existing_path, 0.5)).to be_falsey
+    end
+  end
+end
diff --git a/spec/lib/gitlab/git/storage/health_spec.rb b/spec/lib/gitlab/git/storage/health_spec.rb
new file mode 100644
index 00000000000..28b74a0581f
--- /dev/null
+++ b/spec/lib/gitlab/git/storage/health_spec.rb
@@ -0,0 +1,85 @@
+require 'spec_helper'
+
+describe Gitlab::Git::Storage::Health, clean_gitlab_redis_shared_state: true, broken_storage: true do
+  let(:host1_key) { 'storage_accessible:broken:web01' }
+  let(:host2_key) { 'storage_accessible:default:kiq01' }
+
+  def set_in_redis(cache_key, value)
+    Gitlab::Git::Storage.redis.with do |redis|
+      redis.hmset(cache_key, :failure_count, value)
+    end.first
+  end
+
+  describe '.for_failing_storages' do
+    it 'only includes health status for failures' do
+      set_in_redis(host1_key, 10)
+      set_in_redis(host2_key, 0)
+
+      expect(described_class.for_failing_storages.map(&:storage_name))
+        .to contain_exactly('broken')
+    end
+  end
+
+  describe '.load_for_keys' do
+    let(:subject) do
+      results = Gitlab::Git::Storage.redis.with do |redis|
+        described_class.load_for_keys({ 'broken' => [host1_key] }, redis)
+      end
+
+      # Make sure the `Redis#future is loaded
+      results.inject({}) do |result, (name, info)|
+        info.each { |i| i[:failure_count] = i[:failure_count].value.to_i }
+
+        result[name] = info
+
+        result
+      end
+    end
+
+    it 'loads when there is no info in redis' do
+      expect(subject).to eq('broken' => [{ name: host1_key, failure_count: 0 }])
+    end
+
+    it 'reads the correct values for a storage from redis' do
+      set_in_redis(host1_key, 5)
+      set_in_redis(host2_key, 7)
+
+      expect(subject).to eq('broken' => [{ name: host1_key, failure_count: 5 }])
+    end
+  end
+
+  describe '.for_all_storages' do
+    it 'loads health status for all configured storages' do
+      healths = described_class.for_all_storages
+
+      expect(healths.map(&:storage_name)).to contain_exactly('default', 'broken')
+    end
+  end
+
+  describe '#failing_info' do
+    it 'only contains storages that have failures' do
+      health = described_class.new('broken', [{ name: host1_key, failure_count: 0 },
+                                              { name: host2_key, failure_count: 3 }])
+
+      expect(health.failing_info).to contain_exactly({ name: host2_key, failure_count: 3 })
+    end
+  end
+
+  describe '#total_failures' do
+    it 'sums up all the failures' do
+      health = described_class.new('broken', [{ name: host1_key, failure_count: 2 },
+                                              { name: host2_key, failure_count: 3 }])
+
+      expect(health.total_failures).to eq(5)
+    end
+  end
+
+  describe '#failing_on_hosts' do
+    it 'collects only the failing hostnames' do
+      health = described_class.new('broken', [{ name: host1_key, failure_count: 2 },
+                                              { name: host2_key, failure_count: 0 }])
+
+      expect(health.failing_on_hosts).to contain_exactly('web01')
+    end
+  end
+end
diff --git a/spec/lib/gitlab/health_checks/fs_shards_check_spec.rb b/spec/lib/gitlab/health_checks/fs_shards_check_spec.rb
index 8abc4320c59..8539c6deea6 100644
--- a/spec/lib/gitlab/health_checks/fs_shards_check_spec.rb
+++ b/spec/lib/gitlab/health_checks/fs_shards_check_spec.rb
@@ -44,6 +44,15 @@ describe Gitlab::HealthChecks::FsShardsCheck do
     describe '#readiness' do
       subject { described_class.readiness }
 
+      context 'storage has a tripped circuitbreaker' do
+        let(:repository_storages) { ['broken'] }
+        let(:storages_paths) do
+          Gitlab.config.repositories.storages
+        end
+
+        it { is_expected.to include(result_class.new(false, 'circuitbreaker tripped', shard: 'broken')) }
+      end
+
       context 'storage points to not existing folder' do
         let(:storages_paths) do
           {
@@ -51,6 +60,10 @@ describe Gitlab::HealthChecks::FsShardsCheck do
           }.with_indifferent_access
         end
 
+        before do
+          allow(described_class).to receive(:storage_circuitbreaker_test) { true }
+        end
+
         it { is_expected.to include(result_class.new(false, 'cannot stat storage', shard: :default)) }
       end
 
@@ -109,6 +122,7 @@ describe Gitlab::HealthChecks::FsShardsCheck do
           expect(metrics).to include(an_object_having_attributes(name: :filesystem_access_latency_seconds, value: be >= 0))
           expect(metrics).to include(an_object_having_attributes(name: :filesystem_read_latency_seconds, value: be >= 0))
           expect(metrics).to include(an_object_having_attributes(name: :filesystem_write_latency_seconds, value: be >= 0))
+          expect(metrics).to include(an_object_having_attributes(name: :filesystem_circuitbreaker_latency_seconds, value: be >= 0))
         end
       end
 
@@ -127,6 +141,7 @@ describe Gitlab::HealthChecks::FsShardsCheck do
           expect(metrics).to include(an_object_having_attributes(name: :filesystem_access_latency_seconds, value: be >= 0))
           expect(metrics).to include(an_object_having_attributes(name: :filesystem_read_latency_seconds, value: be >= 0))
           expect(metrics).to include(an_object_having_attributes(name: :filesystem_write_latency_seconds, value: be >= 0))
+          expect(metrics).to include(an_object_having_attributes(name: :filesystem_circuitbreaker_latency_seconds, value: be >= 0))
         end
 
         it 'cleans up files used for metrics' do
diff --git a/spec/models/repository_spec.rb b/spec/models/repository_spec.rb
index 764f548be45..d077925a1cf 100644
--- a/spec/models/repository_spec.rb
+++ b/spec/models/repository_spec.rb
@@ -1,11 +1,12 @@
 require 'spec_helper'
 
-describe Repository do
+describe Repository, models: true do
   include RepoHelpers
   TestBlob = Struct.new(:path)
 
   let(:project) { create(:project, :repository) }
   let(:repository) { project.repository }
+  let(:broken_repository) { create(:project, :broken_storage).repository }
   let(:user) { create(:user) }
 
   let(:commit_options) do
@@ -27,12 +28,26 @@ describe Repository do
   let(:author_email) { 'user@example.org' }
   let(:author_name) { 'John Doe' }
 
+  def expect_to_raise_storage_error
+    expect { yield }.to raise_error do |exception|
+      expect(exception.class).to be_in([Gitlab::Git::Storage::Inaccessible, GRPC::Unavailable])
+    end
+  end
+
   describe '#branch_names_contains' do
     subject { repository.branch_names_contains(sample_commit.id) }
 
     it { is_expected.to include('master') }
     it { is_expected.not_to include('feature') }
     it { is_expected.not_to include('fix') }
+
+    describe 'when storage is broken', broken_storage: true  do
+      it 'should raise a storage error' do
+        expect_to_raise_storage_error do
+          broken_repository.branch_names_contains(sample_commit.id)
+        end
+      end
+    end
   end
 
   describe '#tag_names_contains' do
@@ -142,6 +157,14 @@ describe Repository do
     subject { repository.last_commit_for_path(sample_commit.id, '.gitignore').id }
 
     it { is_expected.to eq('c1acaa58bbcbc3eafe538cb8274ba387047b69f8') }
+
+    describe 'when storage is broken', broken_storage: true  do
+      it 'should raise a storage error' do
+        expect_to_raise_storage_error do
+          broken_repository.last_commit_for_path(sample_commit.id, '.gitignore').id
+        end
+      end
+    end
   end
 
   describe '#last_commit_id_for_path' do
@@ -158,6 +181,14 @@ describe Repository do
       expect(cache).to receive(:fetch).with(key).and_return('c1acaa5')
       is_expected.to eq('c1acaa5')
     end
+
+    describe 'when storage is broken', broken_storage: true  do
+      it 'should raise a storage error' do
+        expect_to_raise_storage_error do
+          broken_repository.last_commit_id_for_path(sample_commit.id, '.gitignore')
+        end
+      end
+    end
   end
 
   describe '#commits' do
@@ -196,6 +227,12 @@ describe Repository do
 
       expect(commit_ids).to include('5937ac0a7beb003549fc5fd26fc247adbce4a52e')
     end
+
+    describe 'when storage is broken', broken_storage: true  do
+      it 'should raise a storage error' do
+        expect_to_raise_storage_error { broken_repository.find_commits_by_message('s') }
+      end
+    end
   end
 
   describe '#blob_at' do
@@ -521,6 +558,14 @@ describe Repository do
       expect(results).to match_array([])
     end
 
+    describe 'when storage is broken', broken_storage: true  do
+      it 'should raise a storage error' do
+        expect_to_raise_storage_error do
+          broken_repository.search_files_by_content('feature', 'master')
+        end
+      end
+    end
+
     describe 'result' do
       subject { results.first }
 
@@ -549,6 +594,22 @@ describe Repository do
 
       expect(results).to match_array([])
     end
+
+    describe 'when storage is broken', broken_storage: true  do
+      it 'should raise a storage error' do
+        expect_to_raise_storage_error { broken_repository.search_files_by_name('files', 'master') }
+      end
+    end
+  end
+
+  describe '#fetch_ref' do
+    describe 'when storage is broken', broken_storage: true  do
+      it 'should raise a storage error' do
+        path = broken_repository.path_to_repo
+
+        expect_to_raise_storage_error { broken_repository.fetch_ref(path, '1', '2') }
+      end
+    end
   end
 
   describe '#create_ref' do
@@ -966,6 +1027,12 @@ describe Repository do
 
       expect(repository.exists?).to eq(false)
     end
+
+    context 'with broken storage', broken_storage: true do
+      it 'should raise a storage error' do
+        expect_to_raise_storage_error { broken_repository.exists? }
+      end
+    end
   end
 
   describe '#exists?' do
diff --git a/spec/requests/api/circuit_breakers_spec.rb b/spec/requests/api/circuit_breakers_spec.rb
new file mode 100644
index 00000000000..76521e55994
--- /dev/null
+++ b/spec/requests/api/circuit_breakers_spec.rb
@@ -0,0 +1,57 @@
+require 'spec_helper'
+
+describe API::CircuitBreakers do
+  let(:user) { create(:user) }
+  let(:admin) { create(:admin) }
+
+  describe 'GET circuit_breakers/repository_storage' do
+    it 'returns a 401 for anonymous users' do
+      get api('/circuit_breakers/repository_storage')
+
+      expect(response).to have_http_status(401)
+    end
+
+    it 'returns a 403 for users' do
+      get api('/circuit_breakers/repository_storage', user)
+
+      expect(response).to have_http_status(403)
+    end
+
+    it 'returns an Array of storages' do
+      expect(Gitlab::Git::Storage::Health).to receive(:for_all_storages) do
+        [Gitlab::Git::Storage::Health.new('broken', [{ name: 'prefix:broken:web01', failure_count: 4 }])]
+      end
+
+      get api('/circuit_breakers/repository_storage', admin)
+
+      expect(response).to have_http_status(200)
+      expect(json_response).to be_kind_of(Array)
+      expect(json_response.first['storage_name']).to eq('broken')
+      expect(json_response.first['failing_on_hosts']).to eq(['web01'])
+      expect(json_response.first['total_failures']).to eq(4)
+    end
+
+    describe 'GET circuit_breakers/repository_storage/failing' do
+      it 'returns an array of failing storages' do
+        expect(Gitlab::Git::Storage::Health).to receive(:for_failing_storages) do
+          [Gitlab::Git::Storage::Health.new('broken', [{ name: 'prefix:broken:web01', failure_count: 4 }])]
+        end
+
+        get api('/circuit_breakers/repository_storage/failing', admin)
+
+        expect(response).to have_http_status(200)
+        expect(json_response).to be_kind_of(Array)
+      end
+    end
+  end
+
+  describe 'DELETE circuit_breakers/repository_storage' do
+    it 'clears all circuit_breakers' do
+      expect(Gitlab::Git::Storage::CircuitBreaker).to receive(:reset_all!)
+
+      delete api('/circuit_breakers/repository_storage', admin)
+
+      expect(response).to have_http_status(204)
+    end
+  end
+end
diff --git a/spec/support/stored_repositories.rb b/spec/support/stored_repositories.rb
index df18926d58c..f3deae0f455 100644
--- a/spec/support/stored_repositories.rb
+++ b/spec/support/stored_repositories.rb
@@ -2,4 +2,16 @@ RSpec.configure do |config|
   config.before(:each, :repository) do
     TestEnv.clean_test_path
   end
+
+  config.before(:all, :broken_storage) do
+    FileUtils.rm_rf Gitlab.config.repositories.storages.broken['path']
+  end
+
+  config.before(:each, :broken_storage) do
+    allow(Gitlab::GitalyClient).to receive(:call) do
+      raise GRPC::Unavailable.new('Gitaly broken in this spec')
+    end
+
+    Gitlab::Git::Storage::CircuitBreaker.reset_all!
+  end
 end