diff options
author | Shinya Maeda <shinya@gitlab.com> | 2018-09-25 09:18:02 +0300 |
---|---|---|
committer | Alessio Caiazza <acaiazza@gitlab.com> | 2018-10-02 18:04:04 +0300 |
commit | c9077a0efdca065f848e3698c3afd5251a135049 (patch) | |
tree | 11a616d4415ce72fbba774f54c777c68a3da6bea /app/workers/stuck_ci_jobs_worker.rb | |
parent | f76f6df19377945609c7e68103077ef04b0702fb (diff) |
Add cleanup mechanizm for stale scheduled jobs
Diffstat (limited to 'app/workers/stuck_ci_jobs_worker.rb')
-rw-r--r-- | app/workers/stuck_ci_jobs_worker.rb | 30 |
1 files changed, 17 insertions, 13 deletions
diff --git a/app/workers/stuck_ci_jobs_worker.rb b/app/workers/stuck_ci_jobs_worker.rb index f6bca1176d1..884843e4465 100644 --- a/app/workers/stuck_ci_jobs_worker.rb +++ b/app/workers/stuck_ci_jobs_worker.rb @@ -8,6 +8,7 @@ class StuckCiJobsWorker BUILD_RUNNING_OUTDATED_TIMEOUT = 1.hour BUILD_PENDING_OUTDATED_TIMEOUT = 1.day + BUILD_SCHEDULED_OUTDATED_TIMEOUT = 1.hour BUILD_PENDING_STUCK_TIMEOUT = 1.hour def perform @@ -15,9 +16,10 @@ class StuckCiJobsWorker Rails.logger.info "#{self.class}: Cleaning stuck builds" - drop :running, BUILD_RUNNING_OUTDATED_TIMEOUT - drop :pending, BUILD_PENDING_OUTDATED_TIMEOUT - drop_stuck :pending, BUILD_PENDING_STUCK_TIMEOUT + drop :running, :updated_at, BUILD_RUNNING_OUTDATED_TIMEOUT, :stuck_or_timeout_failure + drop :pending, :updated_at, BUILD_PENDING_OUTDATED_TIMEOUT, :stuck_or_timeout_failure + drop :scheduled, :scheduled_at, BUILD_SCHEDULED_OUTDATED_TIMEOUT, :schedule_expired + drop_stuck :pending, :updated_at, BUILD_PENDING_STUCK_TIMEOUT, :stuck_or_timeout_failure remove_lease end @@ -32,25 +34,27 @@ class StuckCiJobsWorker Gitlab::ExclusiveLease.cancel(EXCLUSIVE_LEASE_KEY, @uuid) end - def drop(status, timeout) - search(status, timeout) do |build| - drop_build :outdated, build, status, timeout + def drop(status, column, timeout, reason) + search(status, column, timeout) do |build| + drop_build :outdated, build, status, timeout, reason end end - def drop_stuck(status, timeout) - search(status, timeout) do |build| + def drop_stuck(status, column, timeout, reason) + search(status, column, timeout) do |build| break unless build.stuck? - drop_build :stuck, build, status, timeout + drop_build :stuck, build, status, timeout, reason end end # rubocop: disable CodeReuse/ActiveRecord - def search(status, timeout) + def search(status, column, timeout) + quoted_column = ActiveRecord::Base.connection.quote_column_name(column) + loop do jobs = Ci::Build.where(status: status) - .where('ci_builds.updated_at < ?', timeout.ago) + .where("#{quoted_column} < ?", timeout.ago) .includes(:tags, :runner, project: :namespace) .limit(100) .to_a @@ -63,10 +67,10 @@ class StuckCiJobsWorker end # rubocop: enable CodeReuse/ActiveRecord - def drop_build(type, build, status, timeout) + def drop_build(type, build, status, timeout, reason) Rails.logger.info "#{self.class}: Dropping #{type} build #{build.id} for runner #{build.runner_id} (status: #{status}, timeout: #{timeout})" Gitlab::OptimisticLocking.retry_lock(build, 3) do |b| - b.drop(:stuck_or_timeout_failure) + b.drop(reason) end end end |