diff options
Diffstat (limited to 'app/workers/gitlab')
7 files changed, 131 insertions, 4 deletions
diff --git a/app/workers/gitlab/github_import/advance_stage_worker.rb b/app/workers/gitlab/github_import/advance_stage_worker.rb index 21e478f935b..834c2f7791c 100644 --- a/app/workers/gitlab/github_import/advance_stage_worker.rb +++ b/app/workers/gitlab/github_import/advance_stage_worker.rb @@ -12,6 +12,7 @@ module Gitlab sidekiq_options dead: false feature_category :importers + loggable_arguments 1, 2 # The known importer stages and their corresponding Sidekiq workers. STAGES = { diff --git a/app/workers/gitlab/import/stuck_import_job.rb b/app/workers/gitlab/import/stuck_import_job.rb new file mode 100644 index 00000000000..16be7a77ab1 --- /dev/null +++ b/app/workers/gitlab/import/stuck_import_job.rb @@ -0,0 +1,82 @@ +# frozen_string_literal: true + +module Gitlab + module Import + module StuckImportJob + extend ActiveSupport::Concern + + IMPORT_JOBS_EXPIRATION = 15.hours.seconds.to_i + + included do + include ApplicationWorker + # rubocop:disable Scalability/CronWorkerContext + # This worker updates several import states inline and does not schedule + # other jobs. So no context needed + include CronjobQueue + # rubocop:enable Scalability/CronWorkerContext + + feature_category :importers + worker_resource_boundary :cpu + end + + def perform + stuck_imports_without_jid_count = mark_imports_without_jid_as_failed! + stuck_imports_with_jid_count = mark_imports_with_jid_as_failed! + + track_metrics(stuck_imports_with_jid_count, stuck_imports_without_jid_count) + end + + private + + def track_metrics(with_jid_count, without_jid_count) + raise NotImplementedError + end + + def mark_imports_without_jid_as_failed! + enqueued_import_states_without_jid.each do |import_state| + import_state.mark_as_failed(error_message) + end.size + end + + def mark_imports_with_jid_as_failed! + jids_and_ids = enqueued_import_states_with_jid.pluck(:jid, :id).to_h # rubocop: disable CodeReuse/ActiveRecord + + # Find the jobs that aren't currently running or that exceeded the threshold. + completed_jids = Gitlab::SidekiqStatus.completed_jids(jids_and_ids.keys) + return 0 unless completed_jids.any? + + completed_import_state_ids = jids_and_ids.values_at(*completed_jids) + + # We select the import states again, because they may have transitioned from + # scheduled/started to finished/failed while we were looking up their Sidekiq status. + completed_import_states = enqueued_import_states_with_jid.id_in(completed_import_state_ids) + completed_import_state_jids = completed_import_states.map { |import_state| import_state.jid }.join(', ') + + Gitlab::Import::Logger.info( + message: 'Marked stuck import jobs as failed', + job_ids: completed_import_state_jids + ) + + completed_import_states.each do |import_state| + import_state.mark_as_failed(error_message) + end.size + end + + def enqueued_import_states + raise NotImplementedError + end + + def enqueued_import_states_with_jid + enqueued_import_states.with_jid + end + + def enqueued_import_states_without_jid + enqueued_import_states.without_jid + end + + def error_message + _("Import timed out. Import took longer than %{import_jobs_expiration} seconds") % { import_jobs_expiration: IMPORT_JOBS_EXPIRATION } + end + end + end +end diff --git a/app/workers/gitlab/import/stuck_project_import_jobs_worker.rb b/app/workers/gitlab/import/stuck_project_import_jobs_worker.rb new file mode 100644 index 00000000000..01979b2029f --- /dev/null +++ b/app/workers/gitlab/import/stuck_project_import_jobs_worker.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true +module Gitlab + module Import + class StuckProjectImportJobsWorker # rubocop:disable Scalability/IdempotentWorker + include Gitlab::Import::StuckImportJob + + private + + def track_metrics(with_jid_count, without_jid_count) + Gitlab::Metrics.add_event( + :stuck_import_jobs, + projects_without_jid_count: without_jid_count, + projects_with_jid_count: with_jid_count + ) + end + + def enqueued_import_states + ProjectImportState.with_status([:scheduled, :started]) + end + end + end +end diff --git a/app/workers/gitlab/jira_import/import_issue_worker.rb b/app/workers/gitlab/jira_import/import_issue_worker.rb index 78de5cf1307..7709d2ec31b 100644 --- a/app/workers/gitlab/jira_import/import_issue_worker.rb +++ b/app/workers/gitlab/jira_import/import_issue_worker.rb @@ -8,6 +8,8 @@ module Gitlab include Gitlab::JiraImport::QueueOptions include Gitlab::Import::DatabaseHelpers + loggable_arguments 3 + def perform(project_id, jira_issue_id, issue_attributes, waiter_key) issue_id = create_issue(issue_attributes, project_id) JiraImport.cache_issue_mapping(issue_id, jira_issue_id, project_id) @@ -48,7 +50,7 @@ module Gitlab label_link_attrs << build_label_attrs(issue_id, import_label_id.to_i) - Gitlab::Database.bulk_insert(LabelLink.table_name, label_link_attrs) + Gitlab::Database.bulk_insert(LabelLink.table_name, label_link_attrs) # rubocop:disable Gitlab/BulkInsert end def assign_issue(project_id, issue_id, assignee_ids) @@ -56,7 +58,7 @@ module Gitlab assignee_attrs = assignee_ids.map { |user_id| { issue_id: issue_id, user_id: user_id } } - Gitlab::Database.bulk_insert(IssueAssignee.table_name, assignee_attrs) + Gitlab::Database.bulk_insert(IssueAssignee.table_name, assignee_attrs) # rubocop:disable Gitlab/BulkInsert end def build_label_attrs(issue_id, label_id) diff --git a/app/workers/gitlab/jira_import/stage/start_import_worker.rb b/app/workers/gitlab/jira_import/stage/start_import_worker.rb index 5b36feadbd1..bfc02224ee4 100644 --- a/app/workers/gitlab/jira_import/stage/start_import_worker.rb +++ b/app/workers/gitlab/jira_import/stage/start_import_worker.rb @@ -25,7 +25,6 @@ module Gitlab def start_import return false unless project - return false unless project.jira_issues_import_feature_flag_enabled? return true if start(project.latest_jira_import) Gitlab::Import::Logger.info( diff --git a/app/workers/gitlab/jira_import/stuck_jira_import_jobs_worker.rb b/app/workers/gitlab/jira_import/stuck_jira_import_jobs_worker.rb new file mode 100644 index 00000000000..5e675193a8c --- /dev/null +++ b/app/workers/gitlab/jira_import/stuck_jira_import_jobs_worker.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module Gitlab + module JiraImport + class StuckJiraImportJobsWorker # rubocop:disable Scalability/IdempotentWorker + include Gitlab::Import::StuckImportJob + + private + + def track_metrics(with_jid_count, without_jid_count) + Gitlab::Metrics.add_event(:stuck_jira_import_jobs, + jira_imports_without_jid_count: with_jid_count, + jira_imports_with_jid_count: without_jid_count) + end + + def enqueued_import_states + JiraImportState.with_status([:scheduled, :started]) + end + end + end +end diff --git a/app/workers/gitlab/phabricator_import/base_worker.rb b/app/workers/gitlab/phabricator_import/base_worker.rb index 82ef9e825f9..2dc4855f854 100644 --- a/app/workers/gitlab/phabricator_import/base_worker.rb +++ b/app/workers/gitlab/phabricator_import/base_worker.rb @@ -13,7 +13,7 @@ # - It keeps track of the jobs so we know how many jobs are running for the # project # - It refreshes the import jid, so it doesn't get cleaned up by the -# `StuckImportJobsWorker` +# `Gitlab::Import::StuckProjectImportJobsWorker` # - It marks the import as failed if a job failed to many times # - It marks the import as finished when all remaining jobs are done module Gitlab |