diff options
Diffstat (limited to 'app/workers')
11 files changed, 278 insertions, 2 deletions
diff --git a/app/workers/all_queues.yml b/app/workers/all_queues.yml index 3df86e3314d..dd0eeaa9359 100644 --- a/app/workers/all_queues.yml +++ b/app/workers/all_queues.yml @@ -528,6 +528,55 @@ :resource_boundary: :unknown :weight: 2 :idempotent: +- :name: jira_importer:jira_import_advance_stage + :feature_category: :importers + :has_external_dependencies: + :urgency: :low + :resource_boundary: :unknown + :weight: 1 + :idempotent: +- :name: jira_importer:jira_import_stage_finish_import + :feature_category: :importers + :has_external_dependencies: + :urgency: :low + :resource_boundary: :unknown + :weight: 1 + :idempotent: +- :name: jira_importer:jira_import_stage_import_attachments + :feature_category: :importers + :has_external_dependencies: + :urgency: :low + :resource_boundary: :unknown + :weight: 1 + :idempotent: +- :name: jira_importer:jira_import_stage_import_issues + :feature_category: :importers + :has_external_dependencies: + :urgency: :low + :resource_boundary: :unknown + :weight: 1 + :idempotent: +- :name: jira_importer:jira_import_stage_import_labels + :feature_category: :importers + :has_external_dependencies: + :urgency: :low + :resource_boundary: :unknown + :weight: 1 + :idempotent: +- :name: jira_importer:jira_import_stage_import_notes + :feature_category: :importers + :has_external_dependencies: + :urgency: :low + :resource_boundary: :unknown + :weight: 1 + :idempotent: +- :name: jira_importer:jira_import_stage_start_import + :feature_category: :importers + :has_external_dependencies: + :urgency: :low + :resource_boundary: :unknown + :weight: 1 + :idempotent: - :name: mail_scheduler:mail_scheduler_issue_due :feature_category: :issue_tracking :has_external_dependencies: diff --git a/app/workers/concerns/gitlab/jira_import/import_worker.rb b/app/workers/concerns/gitlab/jira_import/import_worker.rb new file mode 100644 index 00000000000..7cc650bfc29 --- /dev/null +++ b/app/workers/concerns/gitlab/jira_import/import_worker.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +module Gitlab + module JiraImport + module ImportWorker + extend ActiveSupport::Concern + + included do + include ApplicationWorker + include Gitlab::JiraImport::QueueOptions + end + + def perform(project_id) + project = Project.find_by(id: project_id) # rubocop: disable CodeReuse/ActiveRecord + + return unless can_import?(project) + + import(project) + end + + private + + def import(project) + raise NotImplementedError + end + + def can_import?(project) + return false unless project + return false if Feature.disabled?(:jira_issue_import, project) + + project.import_state.started? + end + end + end +end diff --git a/app/workers/concerns/gitlab/jira_import/queue_options.rb b/app/workers/concerns/gitlab/jira_import/queue_options.rb new file mode 100644 index 00000000000..bc1148f7d3b --- /dev/null +++ b/app/workers/concerns/gitlab/jira_import/queue_options.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +module Gitlab + module JiraImport + module QueueOptions + extend ActiveSupport::Concern + + included do + queue_namespace :jira_importer + feature_category :importers + + sidekiq_options retry: 5 + end + end + end +end diff --git a/app/workers/gitlab/github_import/advance_stage_worker.rb b/app/workers/gitlab/github_import/advance_stage_worker.rb index 8fbf88a1762..8bbfb10ed6e 100644 --- a/app/workers/gitlab/github_import/advance_stage_worker.rb +++ b/app/workers/gitlab/github_import/advance_stage_worker.rb @@ -13,8 +13,6 @@ module Gitlab sidekiq_options dead: false feature_category :importers - private - # The known importer stages and their corresponding Sidekiq workers. STAGES = { issues_and_diff_notes: Stage::ImportIssuesAndDiffNotesWorker, @@ -23,6 +21,8 @@ module Gitlab finish: Stage::FinishImportWorker }.freeze + private + def next_stage_worker(next_stage) STAGES.fetch(next_stage.to_sym) end diff --git a/app/workers/gitlab/jira_import/advance_stage_worker.rb b/app/workers/gitlab/jira_import/advance_stage_worker.rb new file mode 100644 index 00000000000..1b6fc54151e --- /dev/null +++ b/app/workers/gitlab/jira_import/advance_stage_worker.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +module Gitlab + module JiraImport + class AdvanceStageWorker # rubocop:disable Scalability/IdempotentWorker + include ApplicationWorker + include QueueOptions + include ::Gitlab::Import::AdvanceStage + + # The known importer stages and their corresponding Sidekiq workers. + STAGES = { + labels: Gitlab::JiraImport::Stage::ImportLabelsWorker, + issues: Gitlab::JiraImport::Stage::ImportIssuesWorker, + attachments: Gitlab::JiraImport::Stage::ImportAttachmentsWorker, + notes: Gitlab::JiraImport::Stage::ImportNotesWorker, + finish: Gitlab::JiraImport::Stage::FinishImportWorker + }.freeze + + private + + def next_stage_worker(next_stage) + STAGES.fetch(next_stage.to_sym) + end + end + end +end diff --git a/app/workers/gitlab/jira_import/stage/finish_import_worker.rb b/app/workers/gitlab/jira_import/stage/finish_import_worker.rb new file mode 100644 index 00000000000..5b1661d68c6 --- /dev/null +++ b/app/workers/gitlab/jira_import/stage/finish_import_worker.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +module Gitlab + module JiraImport + module Stage + class FinishImportWorker # rubocop:disable Scalability/IdempotentWorker + include Gitlab::JiraImport::ImportWorker + + private + + def import(project) + project.after_import + ensure + project.import_data.becomes(JiraImportData).finish_import! + project.import_data.save! + end + end + end + end +end diff --git a/app/workers/gitlab/jira_import/stage/import_attachments_worker.rb b/app/workers/gitlab/jira_import/stage/import_attachments_worker.rb new file mode 100644 index 00000000000..3b209a279b5 --- /dev/null +++ b/app/workers/gitlab/jira_import/stage/import_attachments_worker.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +module Gitlab + module JiraImport + module Stage + class ImportAttachmentsWorker # rubocop:disable Scalability/IdempotentWorker + include Gitlab::JiraImport::ImportWorker + + private + + def import(project) + # fake a attahcments import workers for now. + # new job waiter will have zero jobs_remaining by default, so it will just pass on to next stage + fake_waiter = JobWaiter.new + + project.import_state.refresh_jid_expiration + Gitlab::JiraImport::AdvanceStageWorker.perform_async(project.id, { fake_waiter.key => fake_waiter.jobs_remaining }, :notes) + end + end + end + end +end diff --git a/app/workers/gitlab/jira_import/stage/import_issues_worker.rb b/app/workers/gitlab/jira_import/stage/import_issues_worker.rb new file mode 100644 index 00000000000..79ed8e1f2da --- /dev/null +++ b/app/workers/gitlab/jira_import/stage/import_issues_worker.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +module Gitlab + module JiraImport + module Stage + class ImportIssuesWorker # rubocop:disable Scalability/IdempotentWorker + include Gitlab::JiraImport::ImportWorker + + private + + def import(project) + # fake issues import workers for now + # new job waiter will have zero jobs_remaining by default, so it will just pass on to next stage + jobs_waiter = JobWaiter.new + project.import_state.refresh_jid_expiration + + Gitlab::JiraImport::AdvanceStageWorker.perform_async(project.id, { jobs_waiter.key => jobs_waiter.jobs_remaining }, :attachments) + end + end + end + end +end diff --git a/app/workers/gitlab/jira_import/stage/import_labels_worker.rb b/app/workers/gitlab/jira_import/stage/import_labels_worker.rb new file mode 100644 index 00000000000..b96bb1bbdda --- /dev/null +++ b/app/workers/gitlab/jira_import/stage/import_labels_worker.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +module Gitlab + module JiraImport + module Stage + class ImportLabelsWorker # rubocop:disable Scalability/IdempotentWorker + include Gitlab::JiraImport::ImportWorker + + private + + def import(project) + # fake labels import workers for now + # new job waiter will have zero jobs_remaining by default, so it will just pass on to next stage + fake_waiter = JobWaiter.new + Gitlab::JiraImport::AdvanceStageWorker.perform_async(project.id, { fake_waiter.key => fake_waiter.jobs_remaining }, :issues) + end + end + end + end +end diff --git a/app/workers/gitlab/jira_import/stage/import_notes_worker.rb b/app/workers/gitlab/jira_import/stage/import_notes_worker.rb new file mode 100644 index 00000000000..9eef0d31a8c --- /dev/null +++ b/app/workers/gitlab/jira_import/stage/import_notes_worker.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +module Gitlab + module JiraImport + module Stage + class ImportNotesWorker # rubocop:disable Scalability/IdempotentWorker + include Gitlab::JiraImport::ImportWorker + + private + + def import(project) + # fake notes import workers for now + # new job waiter will have zero jobs_remaining by default, so it will just pass on to next stage + jobs_waiter = JobWaiter.new + project.import_state.refresh_jid_expiration + + Gitlab::JiraImport::AdvanceStageWorker.perform_async(project.id, { jobs_waiter.key => jobs_waiter.jobs_remaining }, :finish) + end + end + end + end +end diff --git a/app/workers/gitlab/jira_import/stage/start_import_worker.rb b/app/workers/gitlab/jira_import/stage/start_import_worker.rb new file mode 100644 index 00000000000..8abbfab647b --- /dev/null +++ b/app/workers/gitlab/jira_import/stage/start_import_worker.rb @@ -0,0 +1,44 @@ +# frozen_string_literal: true + +module Gitlab + module JiraImport + module Stage + class StartImportWorker # rubocop:disable Scalability/IdempotentWorker + include ApplicationWorker + include ProjectStartImport + include ProjectImportOptions + include Gitlab::JiraImport::QueueOptions + + attr_reader :project + + def perform(project_id) + @project = Project.find_by(id: project_id) # rubocop: disable CodeReuse/ActiveRecord + + return unless start_import + + Gitlab::Import::SetAsyncJid.set_jid(project) + + Gitlab::JiraImport::Stage::ImportLabelsWorker.perform_async(project.id) + end + + private + + def start_import + return false unless project + return false if Feature.disabled?(:jira_issue_import, project) + return true if start(project.import_state) + + Gitlab::Import::Logger.info( + { + project_id: project.id, + project_path: project.full_path, + state: project&.import_status, + message: 'inconsistent state while importing' + } + ) + false + end + end + end + end +end |