Merge branch 'feature/gb/auto-retry-failed-ci-job' into 'master'

Make it possible to auto retry a failed CI/CD job

Closes #3442

See merge request !12909

6 files changed, 181 insertions, 1 deletions

diff --git a/spec/factories/ci/builds.rb b/spec/factories/ci/builds.rb
index a77f01ecb00..678cebe365b 100644
--- a/spec/factories/ci/builds.rb
+++ b/spec/factories/ci/builds.rb
@@ -84,6 +84,10 @@ FactoryGirl.define do
       success
     end
 
+    trait :retried do
+      retried true
+    end
+
     trait :cancelable do
       pending
     end
diff --git a/spec/lib/ci/gitlab_ci_yaml_processor_spec.rb b/spec/lib/ci/gitlab_ci_yaml_processor_spec.rb
index ea79389e67e..ed571a2ba05 100644
--- a/spec/lib/ci/gitlab_ci_yaml_processor_spec.rb
+++ b/spec/lib/ci/gitlab_ci_yaml_processor_spec.rb
@@ -32,6 +32,28 @@ module Ci
         end
       end
 
+      describe 'retry entry' do
+        context 'when retry count is specified' do
+          let(:config) do
+            YAML.dump(rspec: { script: 'rspec', retry: 1 })
+          end
+
+          it 'includes retry count in build options attribute' do
+            expect(subject[:options]).to include(retry: 1)
+          end
+        end
+
+        context 'when retry count is not specified' do
+          let(:config) do
+            YAML.dump(rspec: { script: 'rspec' })
+          end
+
+          it 'does not persist retry count in the database' do
+            expect(subject[:options]).not_to have_key(:retry)
+          end
+        end
+      end
+
       describe 'allow failure entry' do
         context 'when job is a manual action' do
           context 'when allow_failure is defined' do
diff --git a/spec/lib/gitlab/ci/config/entry/job_spec.rb b/spec/lib/gitlab/ci/config/entry/job_spec.rb
index c5cad887b64..6769f64f950 100644
--- a/spec/lib/gitlab/ci/config/entry/job_spec.rb
+++ b/spec/lib/gitlab/ci/config/entry/job_spec.rb
@@ -80,6 +80,45 @@ describe Gitlab::Ci::Config::Entry::Job do
           expect(entry.errors).to include "job script can't be blank"
         end
       end
+
+      context 'when retry value is not correct' do
+        context 'when it is not a numeric value' do
+          let(:config) { { retry: true } }
+
+          it 'returns error about invalid type' do
+            expect(entry).not_to be_valid
+            expect(entry.errors).to include 'job retry is not a number'
+          end
+        end
+
+        context 'when it is lower than zero' do
+          let(:config) { { retry: -1 } }
+
+          it 'returns error about value too low' do
+            expect(entry).not_to be_valid
+            expect(entry.errors)
+              .to include 'job retry must be greater than or equal to 0'
+          end
+        end
+
+        context 'when it is not an integer' do
+          let(:config) { { retry: 1.5 } }
+
+          it 'returns error about wrong value' do
+            expect(entry).not_to be_valid
+            expect(entry.errors).to include 'job retry must be an integer'
+          end
+        end
+
+        context 'when the value is too high' do
+          let(:config) { { retry: 10 } }
+
+          it 'returns error about value too high' do
+            expect(entry).not_to be_valid
+            expect(entry.errors).to include 'job retry must be less than or equal to 2'
+          end
+        end
+      end
     end
   end
 
diff --git a/spec/models/ci/build_spec.rb b/spec/models/ci/build_spec.rb
index 154b6759f46..0b521d720f3 100644
--- a/spec/models/ci/build_spec.rb
+++ b/spec/models/ci/build_spec.rb
@@ -802,6 +802,47 @@ describe Ci::Build, :models do
     end
   end
 
+  describe 'build auto retry feature' do
+    describe '#retries_count' do
+      subject { create(:ci_build, name: 'test', pipeline: pipeline) }
+
+      context 'when build has been retried several times' do
+        before do
+          create(:ci_build, :retried, name: 'test', pipeline: pipeline)
+          create(:ci_build, :retried, name: 'test', pipeline: pipeline)
+        end
+
+        it 'reports a correct retry count value' do
+          expect(subject.retries_count).to eq 2
+        end
+      end
+
+      context 'when build has not been retried' do
+        it 'returns zero' do
+          expect(subject.retries_count).to eq 0
+        end
+      end
+    end
+
+    describe '#retries_max' do
+      context 'when max retries value is defined' do
+        subject { create(:ci_build, options: { retry: 1 }) }
+
+        it 'returns a number of configured max retries' do
+          expect(subject.retries_max).to eq 1
+        end
+      end
+
+      context 'when max retries value is not defined' do
+        subject { create(:ci_build) }
+
+        it 'returns zero' do
+          expect(subject.retries_max).to eq 0
+        end
+      end
+    end
+  end
+
   describe '#keep_artifacts!' do
     let(:build) { create(:ci_build, artifacts_expire_at: Time.now + 7.days) }
 
@@ -1583,7 +1624,7 @@ describe Ci::Build, :models do
     end
   end
 
-  describe 'State transition: any => [:pending]' do
+  describe 'state transition: any => [:pending]' do
     let(:build) { create(:ci_build, :created) }
 
     it 'queues BuildQueueWorker' do
@@ -1592,4 +1633,35 @@ describe Ci::Build, :models do
       build.enqueue
     end
   end
+
+  describe 'state transition when build fails' do
+    context 'when build is configured to be retried' do
+      subject { create(:ci_build, :running, options: { retry: 3 }) }
+
+      it 'retries builds and assigns a same user to it' do
+        expect(described_class).to receive(:retry)
+          .with(subject, subject.user)
+
+        subject.drop!
+      end
+    end
+
+    context 'when build is not configured to be retried' do
+      subject { create(:ci_build, :running) }
+
+      it 'does not retry build' do
+        expect(described_class).not_to receive(:retry)
+
+        subject.drop!
+      end
+
+      it 'does not count retries when not necessary' do
+        expect(described_class).not_to receive(:retry)
+        expect_any_instance_of(described_class)
+          .not_to receive(:retries_count)
+
+        subject.drop!
+      end
+    end
+  end
 end
diff --git a/spec/services/ci/create_pipeline_service_spec.rb b/spec/services/ci/create_pipeline_service_spec.rb
index e71c462b99a..ba07c01d43f 100644
--- a/spec/services/ci/create_pipeline_service_spec.rb
+++ b/spec/services/ci/create_pipeline_service_spec.rb
@@ -320,5 +320,19 @@ describe Ci::CreatePipelineService, :services do
         end.not_to change { Environment.count }
       end
     end
+
+    context 'when builds with auto-retries are configured' do
+      before do
+        config = YAML.dump(rspec: { script: 'rspec', retry: 2 })
+        stub_ci_pipeline_yaml_file(config)
+      end
+
+      it 'correctly creates builds with auto-retry value configured' do
+        pipeline = execute_service
+
+        expect(pipeline).to be_persisted
+        expect(pipeline.builds.find_by(name: 'rspec').retries_max).to eq 2
+      end
+    end
   end
 end
diff --git a/spec/services/ci/process_pipeline_service_spec.rb b/spec/services/ci/process_pipeline_service_spec.rb
index efcaccc254e..0934833a4fa 100644
--- a/spec/services/ci/process_pipeline_service_spec.rb
+++ b/spec/services/ci/process_pipeline_service_spec.rb
@@ -463,6 +463,35 @@ describe Ci::ProcessPipelineService, '#execute', :services do
     end
   end
 
+  context 'when builds with auto-retries are configured' do
+    before do
+      create_build('build:1', stage_idx: 0, user: user, options: { retry: 2 })
+      create_build('test:1', stage_idx: 1, user: user, when: :on_failure)
+      create_build('test:2', stage_idx: 1, user: user, options: { retry: 1 })
+    end
+
+    it 'automatically retries builds in a valid order' do
+      expect(process_pipeline).to be_truthy
+
+      fail_running_or_pending
+
+      expect(builds_names).to eq %w[build:1 build:1]
+      expect(builds_statuses).to eq %w[failed pending]
+
+      succeed_running_or_pending
+
+      expect(builds_names).to eq %w[build:1 build:1 test:2]
+      expect(builds_statuses).to eq %w[failed success pending]
+
+      succeed_running_or_pending
+
+      expect(builds_names).to eq %w[build:1 build:1 test:2]
+      expect(builds_statuses).to eq %w[failed success success]
+
+      expect(pipeline.reload).to be_success
+    end
+  end
+
   def process_pipeline
     described_class.new(pipeline.project, user).execute(pipeline)
   end