diff options
-rw-r--r-- | app/workers/stuck_ci_builds_worker.rb | 13 | ||||
-rw-r--r-- | spec/workers/stuck_ci_builds_worker_spec.rb | 17 |
2 files changed, 27 insertions, 3 deletions
diff --git a/app/workers/stuck_ci_builds_worker.rb b/app/workers/stuck_ci_builds_worker.rb index b9dac807b23..4eba311d063 100644 --- a/app/workers/stuck_ci_builds_worker.rb +++ b/app/workers/stuck_ci_builds_worker.rb @@ -7,7 +7,9 @@ class StuckCiBuildsWorker BUILD_PENDING_STUCK_TIMEOUT = 1.hour def perform - Rails.logger.info 'Cleaning stuck builds' + return unless try_obtain_lease + + Rails.logger.info "#{self.class}: Cleaning stuck builds" drop :running, BUILD_RUNNING_OUTDATED_TIMEOUT drop :pending, BUILD_PENDING_OUTDATED_TIMEOUT @@ -16,6 +18,13 @@ class StuckCiBuildsWorker private + def try_obtain_lease + Gitlab::ExclusiveLease.new( + 'stuck_ci_builds_worker_lease', + timeout: 30.minutes + ).try_obtain + end + def drop(status, timeout) search(status, timeout) do |build| drop_build :outdated, build, status, timeout @@ -37,7 +46,7 @@ class StuckCiBuildsWorker end def drop_build(type, build, status, timeout) - Rails.logger.info "#{self.class}: Dropping #{type.to_s} build #{build.id} for runner #{build.runner_id} (status: #{status}, timeout: #{timeout})" + Rails.logger.info "#{self.class}: Dropping #{type} build #{build.id} for runner #{build.runner_id} (status: #{status}, timeout: #{timeout})" build.drop end end diff --git a/spec/workers/stuck_ci_builds_worker_spec.rb b/spec/workers/stuck_ci_builds_worker_spec.rb index e2386336447..c88a0dfa4a6 100644 --- a/spec/workers/stuck_ci_builds_worker_spec.rb +++ b/spec/workers/stuck_ci_builds_worker_spec.rb @@ -10,7 +10,10 @@ describe StuckCiBuildsWorker do build.status end - before { build.update!(status: status, updated_at: updated_at) } + before do + build.update!(status: status, updated_at: updated_at) + allow_any_instance_of(Gitlab::ExclusiveLease).to receive(:try_obtain).and_return(true) + end shared_examples 'build is dropped' do it 'changes status' do @@ -96,4 +99,16 @@ describe StuckCiBuildsWorker do worker.perform end end + + describe 'exclusive lease' do + let(:status) { 'running' } + let(:updated_at) { 2.days.ago } + + it 'is guard by exclusive lease' do + worker.perform + allow_any_instance_of(Gitlab::ExclusiveLease).to receive(:try_obtain).and_return(false) + expect(worker).not_to receive(:drop) + worker.perform + end + end end |