diff options
Diffstat (limited to 'spec/workers/stuck_ci_jobs_worker_spec.rb')
| -rw-r--r-- | spec/workers/stuck_ci_jobs_worker_spec.rb | 299 |
1 files changed, 19 insertions, 280 deletions
diff --git a/spec/workers/stuck_ci_jobs_worker_spec.rb b/spec/workers/stuck_ci_jobs_worker_spec.rb index 84b2d87494e..e0a5d3c6c1c 100644 --- a/spec/workers/stuck_ci_jobs_worker_spec.rb +++ b/spec/workers/stuck_ci_jobs_worker_spec.rb @@ -5,311 +5,50 @@ require 'spec_helper' RSpec.describe StuckCiJobsWorker do include ExclusiveLeaseHelpers - let!(:runner) { create :ci_runner } - let!(:job) { create :ci_build, runner: runner } - let(:worker_lease_key) { StuckCiJobsWorker::EXCLUSIVE_LEASE_KEY } + let(:worker_lease_key) { StuckCiJobsWorker::EXCLUSIVE_LEASE_KEY } let(:worker_lease_uuid) { SecureRandom.uuid } - let(:created_at) { } - let(:updated_at) { } + let(:worker2) { described_class.new } subject(:worker) { described_class.new } before do stub_exclusive_lease(worker_lease_key, worker_lease_uuid) - job_attributes = { status: status } - job_attributes[:created_at] = created_at if created_at - job_attributes[:updated_at] = updated_at if updated_at - job.update!(job_attributes) end - shared_examples 'job is dropped' do - it "changes status" do - worker.perform - job.reload - - expect(job).to be_failed - expect(job).to be_stuck_or_timeout_failure - end - - context 'when job have data integrity problem' do - it "does drop the job and logs the reason" do - job.update_columns(yaml_variables: '[{"key" => "value"}]') - - expect(Gitlab::ErrorTracking).to receive(:track_exception) - .with(anything, a_hash_including(build_id: job.id)) - .once - .and_call_original - - worker.perform - job.reload - - expect(job).to be_failed - expect(job).to be_data_integrity_failure + describe '#perform' do + it 'executes an instance of Ci::StuckBuildsDropService' do + expect_next_instance_of(Ci::StuckBuilds::DropService) do |service| + expect(service).to receive(:execute).exactly(:once) end - end - end - shared_examples 'job is unchanged' do - before do worker.perform - job.reload end - it "doesn't change status" do - expect(job.status).to eq(status) - end - end - - context 'when job is pending' do - let(:status) { 'pending' } - - context 'when job is not stuck' do - before do - allow_any_instance_of(Ci::Build).to receive(:stuck?).and_return(false) - end - - context 'when job was updated_at more than 1 day ago' do - let(:updated_at) { 1.5.days.ago } - - context 'when created_at is the same as updated_at' do - let(:created_at) { 1.5.days.ago } - - it_behaves_like 'job is dropped' - end - - context 'when created_at is before updated_at' do - let(:created_at) { 3.days.ago } - - it_behaves_like 'job is dropped' - end - - context 'when created_at is outside lookback window' do - let(:created_at) { described_class::BUILD_LOOKBACK - 1.day } - - it_behaves_like 'job is unchanged' - end - end - - context 'when job was updated less than 1 day ago' do - let(:updated_at) { 6.hours.ago } - - context 'when created_at is the same as updated_at' do - let(:created_at) { 1.5.days.ago } - - it_behaves_like 'job is unchanged' - end - - context 'when created_at is before updated_at' do - let(:created_at) { 3.days.ago } - - it_behaves_like 'job is unchanged' - end - - context 'when created_at is outside lookback window' do - let(:created_at) { described_class::BUILD_LOOKBACK - 1.day } - - it_behaves_like 'job is unchanged' - end - end - - context 'when job was updated more than 1 hour ago' do - let(:updated_at) { 2.hours.ago } - - context 'when created_at is the same as updated_at' do - let(:created_at) { 2.hours.ago } - - it_behaves_like 'job is unchanged' - end - - context 'when created_at is before updated_at' do - let(:created_at) { 3.days.ago } - - it_behaves_like 'job is unchanged' - end - - context 'when created_at is outside lookback window' do - let(:created_at) { described_class::BUILD_LOOKBACK - 1.day } - - it_behaves_like 'job is unchanged' - end - end - end - - context 'when job is stuck' do - before do - allow_any_instance_of(Ci::Build).to receive(:stuck?).and_return(true) - end - - context 'when job was updated_at more than 1 hour ago' do - let(:updated_at) { 1.5.hours.ago } - - context 'when created_at is the same as updated_at' do - let(:created_at) { 1.5.hours.ago } - - it_behaves_like 'job is dropped' - end - - context 'when created_at is before updated_at' do - let(:created_at) { 3.days.ago } - - it_behaves_like 'job is dropped' - end - - context 'when created_at is outside lookback window' do - let(:created_at) { described_class::BUILD_LOOKBACK - 1.day } - - it_behaves_like 'job is unchanged' - end - end - - context 'when job was updated in less than 1 hour ago' do - let(:updated_at) { 30.minutes.ago } - - context 'when created_at is the same as updated_at' do - let(:created_at) { 30.minutes.ago } - - it_behaves_like 'job is unchanged' - end - - context 'when created_at is before updated_at' do - let(:created_at) { 2.days.ago } - - it_behaves_like 'job is unchanged' - end + context 'with an exclusive lease' do + it 'does not execute concurrently' do + expect(worker).to receive(:remove_lease).exactly(:once) + expect(worker2).not_to receive(:remove_lease) - context 'when created_at is outside lookback window' do - let(:created_at) { described_class::BUILD_LOOKBACK - 1.day } - - it_behaves_like 'job is unchanged' - end - end - end - end - - context 'when job is running' do - let(:status) { 'running' } - - context 'when job was updated_at more than an hour ago' do - let(:updated_at) { 2.hours.ago } - - it_behaves_like 'job is dropped' - end - - context 'when job was updated in less than 1 hour ago' do - let(:updated_at) { 30.minutes.ago } - - it_behaves_like 'job is unchanged' - end - end - - %w(success skipped failed canceled).each do |status| - context "when job is #{status}" do - let(:status) { status } - let(:updated_at) { 2.days.ago } - - context 'when created_at is the same as updated_at' do - let(:created_at) { 2.days.ago } - - it_behaves_like 'job is unchanged' - end - - context 'when created_at is before updated_at' do - let(:created_at) { 3.days.ago } - - it_behaves_like 'job is unchanged' - end + worker.perform - context 'when created_at is outside lookback window' do - let(:created_at) { described_class::BUILD_LOOKBACK - 1.day } + stub_exclusive_lease_taken(worker_lease_key) - it_behaves_like 'job is unchanged' + worker2.perform end - end - end - - context 'for deleted project' do - let(:status) { 'running' } - let(:updated_at) { 2.days.ago } - - before do - job.project.update!(pending_delete: true) - end - - it 'does drop job' do - expect_any_instance_of(Ci::Build).to receive(:drop).and_call_original - worker.perform - end - end - - describe 'drop stale scheduled builds' do - let(:status) { 'scheduled' } - let(:updated_at) { } - - context 'when scheduled at 2 hours ago but it is not executed yet' do - let!(:job) { create(:ci_build, :scheduled, scheduled_at: 2.hours.ago) } - it 'drops the stale scheduled build' do - expect(Ci::Build.scheduled.count).to eq(1) - expect(job).to be_scheduled + it 'can execute in sequence' do + expect(worker).to receive(:remove_lease).at_least(:once) + expect(worker2).to receive(:remove_lease).at_least(:once) worker.perform - job.reload - - expect(Ci::Build.scheduled.count).to eq(0) - expect(job).to be_failed - expect(job).to be_stale_schedule + worker2.perform end - end - - context 'when scheduled at 30 minutes ago but it is not executed yet' do - let!(:job) { create(:ci_build, :scheduled, scheduled_at: 30.minutes.ago) } - it 'does not drop the stale scheduled build yet' do - expect(Ci::Build.scheduled.count).to eq(1) - expect(job).to be_scheduled + it 'cancels exclusive leases after worker perform' do + expect_to_cancel_exclusive_lease(worker_lease_key, worker_lease_uuid) worker.perform - - expect(Ci::Build.scheduled.count).to eq(1) - expect(job).to be_scheduled - end - end - - context 'when there are no stale scheduled builds' do - it 'does not drop the stale scheduled build yet' do - expect { worker.perform }.not_to raise_error end end end - - describe 'exclusive lease' do - let(:status) { 'running' } - let(:updated_at) { 2.days.ago } - let(:worker2) { described_class.new } - - it 'is guard by exclusive lease when executed concurrently' do - expect(worker).to receive(:drop).at_least(:once).and_call_original - expect(worker2).not_to receive(:drop) - - worker.perform - - stub_exclusive_lease_taken(worker_lease_key) - - worker2.perform - end - - it 'can be executed in sequence' do - expect(worker).to receive(:drop).at_least(:once).and_call_original - expect(worker2).to receive(:drop).at_least(:once).and_call_original - - worker.perform - worker2.perform - end - - it 'cancels exclusive leases after worker perform' do - expect_to_cancel_exclusive_lease(worker_lease_key, worker_lease_uuid) - - worker.perform - end - end end |