Retry failed tests with rspec-retry

Does not retry if tagged :quarantine. Also adds unit tests.

2 files changed, 63 insertions, 0 deletions

diff --git a/qa/spec/spec_helper.rb b/qa/spec/spec_helper.rb
index 20a153f3f63..cbdd6e881b1 100644
--- a/qa/spec/spec_helper.rb
+++ b/qa/spec/spec_helper.rb
@@ -1,4 +1,5 @@
 require_relative '../qa'
+require 'rspec/retry'
 
 %w[helpers shared_examples].each do |d|
   Dir[::File.join(__dir__, d, '**', '*.rb')].each { |f| require f }
@@ -31,6 +32,17 @@ RSpec.configure do |config|
   config.profile_examples = 10
   config.order = :random
   Kernel.srand config.seed
+
+  # show retry status in spec process
+  config.verbose_retry = true
+
+  # show exception that triggers a retry if verbose_retry is set to true
+  config.display_try_failure_messages = true
+
+  config.around do |example|
+    retry_times = example.metadata.keys.include?(:quarantine) ? 1 : 3
+    example.run_with_retry retry: retry_times
+  end
 end
 
 # Skip tests in quarantine unless we explicitly focus on them.
diff --git a/qa/spec/spec_helper_spec.rb b/qa/spec/spec_helper_spec.rb
index 2427999e110..27ec1ec80fe 100644
--- a/qa/spec/spec_helper_spec.rb
+++ b/qa/spec/spec_helper_spec.rb
@@ -28,6 +28,22 @@ describe 'rspec config tests' do
     end
   end
 
+  let(:group_2) do
+    RSpec.describe do
+      before(:all) do
+        @expectations = [1, 2, 3]
+      end
+
+      example 'not in quarantine' do
+        expect(@expectations.shift).to be(3)
+      end
+
+      example 'in quarantine', :quarantine do
+        expect(@expectations.shift).to be(3)
+      end
+    end
+  end
+
   context 'with no tags focussed' do
     before do
       group.run
@@ -301,4 +317,39 @@ describe 'rspec config tests' do
       end
     end
   end
+
+  context 'rspec retry' do
+    context 'in an untagged context' do
+      before do
+        group_2.run
+      end
+
+      it 'should run example :retry times' do
+        examples = group_2.descendant_filtered_examples
+        ex = examples.find { |e| e.description == 'not in quarantine' }
+        expect(ex.execution_result.status).to eq(:passed)
+      end
+    end
+
+    context 'with :quarantine focussed' do
+      before do
+        RSpec.configure do |config|
+          config.inclusion_filter = :quarantine
+        end
+        group_2.run
+      end
+
+      after do
+        RSpec.configure do |config|
+          config.inclusion_filter.clear
+        end
+      end
+
+      it 'should run example once only' do
+        examples = group_2.descendant_filtered_examples
+        ex = examples.find { |e| e.description == 'in quarantine' }
+        expect(ex.execution_result.status).to eq(:failed)
+      end
+    end
+  end
 end