Add latest changes from gitlab-org/gitlab@master

4 files changed, 94 insertions, 1 deletions

diff --git a/app/workers/all_queues.yml b/app/workers/all_queues.yml
index 890f38aa26b..71f7c1bac3c 100644
--- a/app/workers/all_queues.yml
+++ b/app/workers/all_queues.yml
@@ -234,6 +234,13 @@
   :resource_boundary: :cpu
   :weight: 1
   :idempotent: 
+- :name: cronjob:stuck_export_jobs
+  :feature_category: :importers
+  :has_external_dependencies: 
+  :urgency: :default
+  :resource_boundary: :cpu
+  :weight: 1
+  :idempotent: 
 - :name: cronjob:stuck_import_jobs
   :feature_category: :importers
   :has_external_dependencies: 
diff --git a/app/workers/concerns/project_export_options.rb b/app/workers/concerns/project_export_options.rb
new file mode 100644
index 00000000000..e9318c1ba43
--- /dev/null
+++ b/app/workers/concerns/project_export_options.rb
@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module ProjectExportOptions
+  extend ActiveSupport::Concern
+
+  EXPORT_RETRY_COUNT = 3
+
+  included do
+    sidekiq_options retry: EXPORT_RETRY_COUNT, status_expiration: StuckExportJobsWorker::EXPORT_JOBS_EXPIRATION
+
+    # We mark the project export as failed once we have exhausted all retries
+    sidekiq_retries_exhausted do |job|
+      project = Project.find(job['args'][1])
+      # rubocop: disable CodeReuse/ActiveRecord
+      job = project.export_jobs.find_by(jid: job["jid"])
+      # rubocop: enable CodeReuse/ActiveRecord
+
+      if job&.fail_op
+        Sidekiq.logger.info "Job #{job['jid']} for project #{project.id} has been set to failed state"
+      else
+        Sidekiq.logger.error "Failed to set Job #{job['jid']} for project #{project.id} to failed state"
+      end
+    end
+  end
+end
diff --git a/app/workers/project_export_worker.rb b/app/workers/project_export_worker.rb
index eefba6d25c7..aaaf70f09b5 100644
--- a/app/workers/project_export_worker.rb
+++ b/app/workers/project_export_worker.rb
@@ -3,17 +3,24 @@
 class ProjectExportWorker # rubocop:disable Scalability/IdempotentWorker
   include ApplicationWorker
   include ExceptionBacktrace
+  include ProjectExportOptions
 
-  sidekiq_options retry: 3
   feature_category :importers
   worker_resource_boundary :memory
 
   def perform(current_user_id, project_id, after_export_strategy = {}, params = {})
     current_user = User.find(current_user_id)
     project = Project.find(project_id)
+    export_job = project.export_jobs.safe_find_or_create_by(jid: self.jid)
     after_export = build!(after_export_strategy)
 
+    export_job&.start
+
     ::Projects::ImportExport::ExportService.new(project, current_user, params).execute(after_export)
+
+    export_job&.finish
+  rescue ActiveRecord::RecordNotFound, Gitlab::ImportExport::AfterExportStrategyBuilder::StrategyNotFoundError => e
+    logger.error("Failed to export project #{project_id}: #{e.message}")
   end
 
   private
diff --git a/app/workers/stuck_export_jobs_worker.rb b/app/workers/stuck_export_jobs_worker.rb
new file mode 100644
index 00000000000..6d8d60d2fc0
--- /dev/null
+++ b/app/workers/stuck_export_jobs_worker.rb
@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+# rubocop:disable Scalability/IdempotentWorker
+class StuckExportJobsWorker
+  include ApplicationWorker
+  # rubocop:disable Scalability/CronWorkerContext
+  # This worker updates export states inline and does not schedule
+  # other jobs.
+  include CronjobQueue
+  # rubocop:enable Scalability/CronWorkerContext
+
+  feature_category :importers
+  worker_resource_boundary :cpu
+
+  EXPORT_JOBS_EXPIRATION = 6.hours.to_i
+
+  def perform
+    failed_jobs_count = mark_stuck_jobs_as_failed!
+
+    Gitlab::Metrics.add_event(:stuck_export_jobs,
+                              failed_jobs_count: failed_jobs_count)
+  end
+
+  private
+
+  # rubocop: disable CodeReuse/ActiveRecord
+  def mark_stuck_jobs_as_failed!
+    jids_and_ids = enqueued_exports.pluck(:jid, :id).to_h
+
+    completed_jids = Gitlab::SidekiqStatus.completed_jids(jids_and_ids.keys)
+    return unless completed_jids.any?
+
+    completed_ids = jids_and_ids.values_at(*completed_jids)
+
+    # We select the export states again, because they may have transitioned from
+    # started to finished while we were looking up their Sidekiq status.
+    completed_jobs = enqueued_exports.where(id: completed_ids)
+
+    Sidekiq.logger.info(
+      message: 'Marked stuck export jobs as failed',
+      job_ids: completed_jobs.map(&:jid)
+    )
+
+    completed_jobs.each do |job|
+      job.fail_op
+    end.count
+  end
+  # rubocop: enable CodeReuse/ActiveRecord
+
+  def enqueued_exports
+    ProjectExportJob.with_status([:started, :queued])
+  end
+end
+# rubocop:enable Scalability/IdempotentWorker