Welcome to mirror list, hosted at ThFree Co, Russian Federation.

gitlab.com/gitlab-org/gitlab-foss.git - Unnamed repository; edit this file 'description' to name the repository.
summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat (limited to 'app/workers/stuck_import_jobs_worker.rb')
-rw-r--r--app/workers/stuck_import_jobs_worker.rb50
1 files changed, 37 insertions, 13 deletions
diff --git a/app/workers/stuck_import_jobs_worker.rb b/app/workers/stuck_import_jobs_worker.rb
index bfc5e667bb6..f850e459cd9 100644
--- a/app/workers/stuck_import_jobs_worker.rb
+++ b/app/workers/stuck_import_jobs_worker.rb
@@ -2,36 +2,60 @@ class StuckImportJobsWorker
include Sidekiq::Worker
include CronjobQueue
- IMPORT_EXPIRATION = 15.hours.to_i
+ IMPORT_JOBS_EXPIRATION = 15.hours.to_i
def perform
- stuck_projects.find_in_batches(batch_size: 500) do |group|
+ projects_without_jid_count = mark_projects_without_jid_as_failed!
+ projects_with_jid_count = mark_projects_with_jid_as_failed!
+
+ Gitlab::Metrics.add_event(:stuck_import_jobs,
+ projects_without_jid_count: projects_without_jid_count,
+ projects_with_jid_count: projects_with_jid_count)
+ end
+
+ private
+
+ def mark_projects_without_jid_as_failed!
+ started_projects_without_jid.each do |project|
+ project.mark_import_as_failed(error_message)
+ end.count
+ end
+
+ def mark_projects_with_jid_as_failed!
+ completed_jids_count = 0
+
+ started_projects_with_jid.find_in_batches(batch_size: 500) do |group|
jids = group.map(&:import_jid)
# Find the jobs that aren't currently running or that exceeded the threshold.
- completed_jids = Gitlab::SidekiqStatus.completed_jids(jids)
+ completed_jids = Gitlab::SidekiqStatus.completed_jids(jids).to_set
if completed_jids.any?
- completed_ids = group.select { |project| completed_jids.include?(project.import_jid) }.map(&:id)
+ completed_jids_count += completed_jids.count
+ group.each do |project|
+ project.mark_import_as_failed(error_message) if completed_jids.include?(project.import_jid)
+ end
- fail_batch!(completed_jids, completed_ids)
+ Rails.logger.info("Marked stuck import jobs as failed. JIDs: #{completed_jids.to_a.join(', ')}")
end
end
- end
- private
+ completed_jids_count
+ end
- def stuck_projects
- Project.select('id, import_jid').with_import_status(:started).where.not(import_jid: nil)
+ def started_projects
+ Project.with_import_status(:started)
end
- def fail_batch!(completed_jids, completed_ids)
- Project.where(id: completed_ids).update_all(import_status: 'failed', import_error: error_message)
+ def started_projects_with_jid
+ started_projects.where.not(import_jid: nil)
+ end
- Rails.logger.info("Marked stuck import jobs as failed. JIDs: #{completed_jids.join(', ')}")
+ def started_projects_without_jid
+ started_projects.where(import_jid: nil)
end
def error_message
- "Import timed out. Import took longer than #{IMPORT_EXPIRATION} seconds"
+ "Import timed out. Import took longer than #{IMPORT_JOBS_EXPIRATION} seconds"
end
end