diff options
Diffstat (limited to 'app/workers/gitlab')
7 files changed, 151 insertions, 7 deletions
diff --git a/app/workers/gitlab/github_import/advance_stage_worker.rb b/app/workers/gitlab/github_import/advance_stage_worker.rb index 70d18d8004c..fdf4ec6f396 100644 --- a/app/workers/gitlab/github_import/advance_stage_worker.rb +++ b/app/workers/gitlab/github_import/advance_stage_worker.rb @@ -25,6 +25,8 @@ module Gitlab issues_and_diff_notes: Stage::ImportIssuesAndDiffNotesWorker, issue_events: Stage::ImportIssueEventsWorker, notes: Stage::ImportNotesWorker, + attachments: Stage::ImportAttachmentsWorker, + protected_branches: Stage::ImportProtectedBranchesWorker, lfs_objects: Stage::ImportLfsObjectsWorker, finish: Stage::FinishImportWorker }.freeze diff --git a/app/workers/gitlab/github_import/import_protected_branch_worker.rb b/app/workers/gitlab/github_import/import_protected_branch_worker.rb new file mode 100644 index 00000000000..c083d8ee867 --- /dev/null +++ b/app/workers/gitlab/github_import/import_protected_branch_worker.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + class ImportProtectedBranchWorker # rubocop:disable Scalability/IdempotentWorker + include ObjectImporter + + worker_resource_boundary :cpu + + def representation_class + Gitlab::GithubImport::Representation::ProtectedBranch + end + + def importer_class + Importer::ProtectedBranchImporter + end + + def object_type + :protected_branch + end + end + end +end diff --git a/app/workers/gitlab/github_import/import_release_attachments_worker.rb b/app/workers/gitlab/github_import/import_release_attachments_worker.rb new file mode 100644 index 00000000000..c6f45ec1d7c --- /dev/null +++ b/app/workers/gitlab/github_import/import_release_attachments_worker.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + class ImportReleaseAttachmentsWorker # rubocop:disable Scalability/IdempotentWorker + include ObjectImporter + + def representation_class + Representation::ReleaseAttachments + end + + def importer_class + Importer::ReleaseAttachmentsImporter + end + + def object_type + :release_attachment + end + end + end +end diff --git a/app/workers/gitlab/github_import/stage/import_attachments_worker.rb b/app/workers/gitlab/github_import/stage/import_attachments_worker.rb new file mode 100644 index 00000000000..e9086dca503 --- /dev/null +++ b/app/workers/gitlab/github_import/stage/import_attachments_worker.rb @@ -0,0 +1,58 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Stage + class ImportAttachmentsWorker # rubocop:disable Scalability/IdempotentWorker + include ApplicationWorker + + data_consistency :always + + sidekiq_options retry: 5 + include GithubImport::Queue + include StageMethods + + # client - An instance of Gitlab::GithubImport::Client. + # project - An instance of Project. + def import(client, project) + return skip_to_next_stage(project) if feature_disabled?(project) + + waiters = importers.each_with_object({}) do |importer, hash| + waiter = start_importer(project, importer, client) + hash[waiter.key] = waiter.jobs_remaining + end + move_to_next_stage(project, waiters) + end + + private + + # For future issue/mr/milestone/etc attachments importers + def importers + [::Gitlab::GithubImport::Importer::ReleasesAttachmentsImporter] + end + + def start_importer(project, importer, client) + info(project.id, message: "starting importer", importer: importer.name) + importer.new(project, client).execute + end + + def skip_to_next_stage(project) + info(project.id, message: "skipping importer", importer: 'Attachments') + move_to_next_stage(project) + end + + def move_to_next_stage(project, waiters = {}) + AdvanceStageWorker.perform_async( + project.id, + waiters, + :protected_branches + ) + end + + def feature_disabled?(project) + Feature.disabled?(:github_importer_attachments_import, project.group, type: :ops) + end + end + end + end +end diff --git a/app/workers/gitlab/github_import/stage/import_notes_worker.rb b/app/workers/gitlab/github_import/stage/import_notes_worker.rb index 167b3e147a3..b53e31ce40e 100644 --- a/app/workers/gitlab/github_import/stage/import_notes_worker.rb +++ b/app/workers/gitlab/github_import/stage/import_notes_worker.rb @@ -21,11 +21,7 @@ module Gitlab hash[waiter.key] = waiter.jobs_remaining end - AdvanceStageWorker.perform_async( - project.id, - waiters, - :lfs_objects - ) + AdvanceStageWorker.perform_async(project.id, waiters, :attachments) end def importers(project) diff --git a/app/workers/gitlab/github_import/stage/import_protected_branches_worker.rb b/app/workers/gitlab/github_import/stage/import_protected_branches_worker.rb new file mode 100644 index 00000000000..6d6dea10e64 --- /dev/null +++ b/app/workers/gitlab/github_import/stage/import_protected_branches_worker.rb @@ -0,0 +1,45 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Stage + class ImportProtectedBranchesWorker # rubocop:disable Scalability/IdempotentWorker + include ApplicationWorker + + data_consistency :always + + sidekiq_options retry: 3 + include GithubImport::Queue + include StageMethods + + # client - An instance of Gitlab::GithubImport::Client. + # project - An instance of Project. + def import(client, project) + info(project.id, + message: "starting importer", + importer: 'Importer::ProtectedBranchesImporter') + waiter = Importer::ProtectedBranchesImporter + .new(project, client) + .execute + + project.import_state.refresh_jid_expiration + + AdvanceStageWorker.perform_async( + project.id, + { waiter.key => waiter.jobs_remaining }, + :lfs_objects + ) + rescue StandardError => e + Gitlab::Import::ImportFailureService.track( + project_id: project.id, + error_source: self.class.name, + exception: e, + metrics: true + ) + + raise(e) + end + end + end + end +end diff --git a/app/workers/gitlab/jira_import/import_issue_worker.rb b/app/workers/gitlab/jira_import/import_issue_worker.rb index 3824cc1f3ef..eabe988dfc2 100644 --- a/app/workers/gitlab/jira_import/import_issue_worker.rb +++ b/app/workers/gitlab/jira_import/import_issue_worker.rb @@ -15,8 +15,7 @@ module Gitlab loggable_arguments 3 def perform(project_id, jira_issue_id, issue_attributes, waiter_key) - issue_id = create_issue(issue_attributes, project_id) - JiraImport.cache_issue_mapping(issue_id, jira_issue_id, project_id) + create_issue(issue_attributes, project_id) rescue StandardError => ex # Todo: Record jira issue id(or better jira issue key), # so that we can report the list of failed to import issues to the user |