diff options
Diffstat (limited to 'app/workers/gitlab')
8 files changed, 147 insertions, 5 deletions
diff --git a/app/workers/gitlab/bitbucket_import/advance_stage_worker.rb b/app/workers/gitlab/bitbucket_import/advance_stage_worker.rb new file mode 100644 index 00000000000..7f281352a1b --- /dev/null +++ b/app/workers/gitlab/bitbucket_import/advance_stage_worker.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +module Gitlab + module BitbucketImport + # AdvanceStageWorker is a worker used by the BitBucket Importer to wait for a + # number of jobs to complete, without blocking a thread. Once all jobs have + # been completed this worker will advance the import process to the next + # stage. + class AdvanceStageWorker # rubocop:disable Scalability/IdempotentWorker + include ApplicationWorker + include ::Gitlab::Import::AdvanceStage + + data_consistency :delayed + + sidekiq_options dead: false, retry: 3 + + feature_category :importers + + loggable_arguments 1, 2 + + # The known importer stages and their corresponding Sidekiq workers. + STAGES = { + finish: Stage::FinishImportWorker + }.freeze + + def find_import_state(project_id) + ProjectImportState.jid_by(project_id: project_id, status: :started) + end + + private + + def next_stage_worker(next_stage) + STAGES.fetch(next_stage.to_sym) + end + end + end +end diff --git a/app/workers/gitlab/bitbucket_import/import_pull_request_worker.rb b/app/workers/gitlab/bitbucket_import/import_pull_request_worker.rb new file mode 100644 index 00000000000..5b06ddf7079 --- /dev/null +++ b/app/workers/gitlab/bitbucket_import/import_pull_request_worker.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +module Gitlab + module BitbucketImport + class ImportPullRequestWorker # rubocop:disable Scalability/IdempotentWorker + include ObjectImporter + + def importer_class + Importers::PullRequestImporter + end + end + end +end diff --git a/app/workers/gitlab/bitbucket_import/stage/finish_import_worker.rb b/app/workers/gitlab/bitbucket_import/stage/finish_import_worker.rb new file mode 100644 index 00000000000..a1c5f5787be --- /dev/null +++ b/app/workers/gitlab/bitbucket_import/stage/finish_import_worker.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module Gitlab + module BitbucketImport + module Stage + class FinishImportWorker # rubocop:disable Scalability/IdempotentWorker + include StageMethods + + private + + def import(project) + project.after_import + + Gitlab::Import::Metrics.new(:bitbucket_importer, project).track_finished_import + end + end + end + end +end diff --git a/app/workers/gitlab/bitbucket_import/stage/import_pull_requests_worker.rb b/app/workers/gitlab/bitbucket_import/stage/import_pull_requests_worker.rb new file mode 100644 index 00000000000..e1f3b5ab79a --- /dev/null +++ b/app/workers/gitlab/bitbucket_import/stage/import_pull_requests_worker.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +module Gitlab + module BitbucketImport + module Stage + class ImportPullRequestsWorker # rubocop:disable Scalability/IdempotentWorker + include StageMethods + + private + + # project - An instance of Project. + def import(project) + waiter = importer_class.new(project).execute + + project.import_state.refresh_jid_expiration + + AdvanceStageWorker.perform_async( + project.id, + { waiter.key => waiter.jobs_remaining }, + :finish + ) + end + + def importer_class + Importers::PullRequestsImporter + end + end + end + end +end diff --git a/app/workers/gitlab/bitbucket_import/stage/import_repository_worker.rb b/app/workers/gitlab/bitbucket_import/stage/import_repository_worker.rb new file mode 100644 index 00000000000..7c6503ae38f --- /dev/null +++ b/app/workers/gitlab/bitbucket_import/stage/import_repository_worker.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +module Gitlab + module BitbucketImport + module Stage + class ImportRepositoryWorker # rubocop:disable Scalability/IdempotentWorker + include StageMethods + + private + + def import(project) + importer = importer_class.new(project) + + importer.execute + + ImportPullRequestsWorker.perform_async(project.id) + end + + def importer_class + Importers::RepositoryImporter + end + + def abort_on_failure + true + end + end + end + end +end diff --git a/app/workers/gitlab/github_gists_import/import_gist_worker.rb b/app/workers/gitlab/github_gists_import/import_gist_worker.rb index 1f17c98dff9..60e4c8fdad6 100644 --- a/app/workers/gitlab/github_gists_import/import_gist_worker.rb +++ b/app/workers/gitlab/github_gists_import/import_gist_worker.rb @@ -25,7 +25,7 @@ module Gitlab # Gitlab::GithubGistsImport::FinishImportWorker to prevent # the entire import from getting stuck if args.length == 3 && (key = args.last) && key.is_a?(String) - JobWaiter.notify(key, jid) + JobWaiter.notify(key, jid, ttl: Gitlab::Import::JOB_WAITER_TTL) end end @@ -48,7 +48,7 @@ module Gitlab ) end - JobWaiter.notify(notify_key, jid) + JobWaiter.notify(notify_key, jid, ttl: Gitlab::Import::JOB_WAITER_TTL) end rescue StandardError => e log_and_track_error(user_id, e, github_identifiers) diff --git a/app/workers/gitlab/import/advance_stage.rb b/app/workers/gitlab/import/advance_stage.rb index 9fc03efe9d0..5d5abc88388 100644 --- a/app/workers/gitlab/import/advance_stage.rb +++ b/app/workers/gitlab/import/advance_stage.rb @@ -15,7 +15,15 @@ module Gitlab # next_stage - The name of the next stage to start when all jobs have been # completed. def perform(project_id, waiters, next_stage) - return unless import_state = find_import_state(project_id) + import_state = find_import_state(project_id) + + # If the import state is nil the project may have been deleted or the import + # may have failed or been canceled. In this case we tidy up the cache data and no + # longer attempt to advance to the next stage. + if import_state.nil? + clear_waiter_caches(waiters) + return + end new_waiters = wait_for_jobs(waiters) @@ -56,6 +64,12 @@ module Gitlab def next_stage_worker(next_stage) raise NotImplementedError end + + def clear_waiter_caches(waiters) + waiters.each_key do |key| + JobWaiter.delete_key(key) + end + end end end end diff --git a/app/workers/gitlab/jira_import/import_issue_worker.rb b/app/workers/gitlab/jira_import/import_issue_worker.rb index eabe988dfc2..2b676238a37 100644 --- a/app/workers/gitlab/jira_import/import_issue_worker.rb +++ b/app/workers/gitlab/jira_import/import_issue_worker.rb @@ -8,9 +8,9 @@ module Gitlab data_consistency :always sidekiq_options retry: 3 - include NotifyUponDeath include Gitlab::JiraImport::QueueOptions include Gitlab::Import::DatabaseHelpers + include Gitlab::Import::NotifyUponDeath loggable_arguments 3 @@ -27,7 +27,7 @@ module Gitlab JiraImport.increment_issue_failures(project_id) ensure # ensure we notify job waiter that the job has finished - JobWaiter.notify(waiter_key, jid) if waiter_key + JobWaiter.notify(waiter_key, jid, ttl: Gitlab::Import::JOB_WAITER_TTL) if waiter_key end private |