diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2020-03-12 12:09:55 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2020-03-12 12:09:55 +0300 |
commit | 18f7828977b74bf6e5153594a098ef90e773b3b7 (patch) | |
tree | 49cb1e16d5341d773807ee583357ae6eb167d61f /app/workers | |
parent | 8191b1571c017378eac33b3ed296ad5216d0a410 (diff) |
Add latest changes from gitlab-org/gitlab@master
Diffstat (limited to 'app/workers')
-rw-r--r-- | app/workers/all_queues.yml | 7 | ||||
-rw-r--r-- | app/workers/concerns/project_export_options.rb | 25 | ||||
-rw-r--r-- | app/workers/project_export_worker.rb | 9 | ||||
-rw-r--r-- | app/workers/stuck_export_jobs_worker.rb | 54 |
4 files changed, 94 insertions, 1 deletions
diff --git a/app/workers/all_queues.yml b/app/workers/all_queues.yml index 890f38aa26b..71f7c1bac3c 100644 --- a/app/workers/all_queues.yml +++ b/app/workers/all_queues.yml @@ -234,6 +234,13 @@ :resource_boundary: :cpu :weight: 1 :idempotent: +- :name: cronjob:stuck_export_jobs + :feature_category: :importers + :has_external_dependencies: + :urgency: :default + :resource_boundary: :cpu + :weight: 1 + :idempotent: - :name: cronjob:stuck_import_jobs :feature_category: :importers :has_external_dependencies: diff --git a/app/workers/concerns/project_export_options.rb b/app/workers/concerns/project_export_options.rb new file mode 100644 index 00000000000..e9318c1ba43 --- /dev/null +++ b/app/workers/concerns/project_export_options.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module ProjectExportOptions + extend ActiveSupport::Concern + + EXPORT_RETRY_COUNT = 3 + + included do + sidekiq_options retry: EXPORT_RETRY_COUNT, status_expiration: StuckExportJobsWorker::EXPORT_JOBS_EXPIRATION + + # We mark the project export as failed once we have exhausted all retries + sidekiq_retries_exhausted do |job| + project = Project.find(job['args'][1]) + # rubocop: disable CodeReuse/ActiveRecord + job = project.export_jobs.find_by(jid: job["jid"]) + # rubocop: enable CodeReuse/ActiveRecord + + if job&.fail_op + Sidekiq.logger.info "Job #{job['jid']} for project #{project.id} has been set to failed state" + else + Sidekiq.logger.error "Failed to set Job #{job['jid']} for project #{project.id} to failed state" + end + end + end +end diff --git a/app/workers/project_export_worker.rb b/app/workers/project_export_worker.rb index eefba6d25c7..aaaf70f09b5 100644 --- a/app/workers/project_export_worker.rb +++ b/app/workers/project_export_worker.rb @@ -3,17 +3,24 @@ class ProjectExportWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker include ExceptionBacktrace + include ProjectExportOptions - sidekiq_options retry: 3 feature_category :importers worker_resource_boundary :memory def perform(current_user_id, project_id, after_export_strategy = {}, params = {}) current_user = User.find(current_user_id) project = Project.find(project_id) + export_job = project.export_jobs.safe_find_or_create_by(jid: self.jid) after_export = build!(after_export_strategy) + export_job&.start + ::Projects::ImportExport::ExportService.new(project, current_user, params).execute(after_export) + + export_job&.finish + rescue ActiveRecord::RecordNotFound, Gitlab::ImportExport::AfterExportStrategyBuilder::StrategyNotFoundError => e + logger.error("Failed to export project #{project_id}: #{e.message}") end private diff --git a/app/workers/stuck_export_jobs_worker.rb b/app/workers/stuck_export_jobs_worker.rb new file mode 100644 index 00000000000..6d8d60d2fc0 --- /dev/null +++ b/app/workers/stuck_export_jobs_worker.rb @@ -0,0 +1,54 @@ +# frozen_string_literal: true + +# rubocop:disable Scalability/IdempotentWorker +class StuckExportJobsWorker + include ApplicationWorker + # rubocop:disable Scalability/CronWorkerContext + # This worker updates export states inline and does not schedule + # other jobs. + include CronjobQueue + # rubocop:enable Scalability/CronWorkerContext + + feature_category :importers + worker_resource_boundary :cpu + + EXPORT_JOBS_EXPIRATION = 6.hours.to_i + + def perform + failed_jobs_count = mark_stuck_jobs_as_failed! + + Gitlab::Metrics.add_event(:stuck_export_jobs, + failed_jobs_count: failed_jobs_count) + end + + private + + # rubocop: disable CodeReuse/ActiveRecord + def mark_stuck_jobs_as_failed! + jids_and_ids = enqueued_exports.pluck(:jid, :id).to_h + + completed_jids = Gitlab::SidekiqStatus.completed_jids(jids_and_ids.keys) + return unless completed_jids.any? + + completed_ids = jids_and_ids.values_at(*completed_jids) + + # We select the export states again, because they may have transitioned from + # started to finished while we were looking up their Sidekiq status. + completed_jobs = enqueued_exports.where(id: completed_ids) + + Sidekiq.logger.info( + message: 'Marked stuck export jobs as failed', + job_ids: completed_jobs.map(&:jid) + ) + + completed_jobs.each do |job| + job.fail_op + end.count + end + # rubocop: enable CodeReuse/ActiveRecord + + def enqueued_exports + ProjectExportJob.with_status([:started, :queued]) + end +end +# rubocop:enable Scalability/IdempotentWorker |