diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2023-05-25 22:51:41 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2023-05-25 22:51:41 +0300 |
commit | e7b32a77cc40a14eb0dd6ae1bfc6f036819c66cc (patch) | |
tree | 86f33d8f98e4eab9c8a8e900f9e370939921c378 /app | |
parent | 4711b9334036cc4719fb9e475545709e8fd5b649 (diff) |
Add latest changes from gitlab-org/gitlab@16-0-stable-ee
Diffstat (limited to 'app')
12 files changed, 472 insertions, 0 deletions
diff --git a/app/workers/all_queues.yml b/app/workers/all_queues.yml index 50b301b2fc3..17cc7bb73c2 100644 --- a/app/workers/all_queues.yml +++ b/app/workers/all_queues.yml @@ -2316,6 +2316,87 @@ :weight: 1 :idempotent: false :tags: [] +- :name: bitbucket_server_import_advance_stage + :worker_name: Gitlab::BitbucketServerImport::AdvanceStageWorker + :feature_category: :importers + :has_external_dependencies: false + :urgency: :low + :resource_boundary: :unknown + :weight: 1 + :idempotent: false + :tags: [] +- :name: bitbucket_server_import_import_lfs_object + :worker_name: Gitlab::BitbucketServerImport::ImportLfsObjectWorker + :feature_category: :importers + :has_external_dependencies: true + :urgency: :low + :resource_boundary: :unknown + :weight: 1 + :idempotent: false + :tags: [] +- :name: bitbucket_server_import_import_pull_request + :worker_name: Gitlab::BitbucketServerImport::ImportPullRequestWorker + :feature_category: :importers + :has_external_dependencies: true + :urgency: :low + :resource_boundary: :unknown + :weight: 1 + :idempotent: false + :tags: [] +- :name: bitbucket_server_import_import_pull_request_notes + :worker_name: Gitlab::BitbucketServerImport::ImportPullRequestNotesWorker + :feature_category: :importers + :has_external_dependencies: true + :urgency: :low + :resource_boundary: :unknown + :weight: 1 + :idempotent: false + :tags: [] +- :name: bitbucket_server_import_stage_finish_import + :worker_name: Gitlab::BitbucketServerImport::Stage::FinishImportWorker + :feature_category: :importers + :has_external_dependencies: true + :urgency: :low + :resource_boundary: :unknown + :weight: 1 + :idempotent: false + :tags: [] +- :name: bitbucket_server_import_stage_import_lfs_objects + :worker_name: Gitlab::BitbucketServerImport::Stage::ImportLfsObjectsWorker + :feature_category: :importers + :has_external_dependencies: true + :urgency: :low + :resource_boundary: :unknown + :weight: 1 + :idempotent: false + :tags: [] +- :name: bitbucket_server_import_stage_import_notes + :worker_name: Gitlab::BitbucketServerImport::Stage::ImportNotesWorker + :feature_category: :importers + :has_external_dependencies: true + :urgency: :low + :resource_boundary: :unknown + :weight: 1 + :idempotent: false + :tags: [] +- :name: bitbucket_server_import_stage_import_pull_requests + :worker_name: Gitlab::BitbucketServerImport::Stage::ImportPullRequestsWorker + :feature_category: :importers + :has_external_dependencies: true + :urgency: :low + :resource_boundary: :unknown + :weight: 1 + :idempotent: false + :tags: [] +- :name: bitbucket_server_import_stage_import_repository + :worker_name: Gitlab::BitbucketServerImport::Stage::ImportRepositoryWorker + :feature_category: :importers + :has_external_dependencies: true + :urgency: :low + :resource_boundary: :unknown + :weight: 1 + :idempotent: false + :tags: [] - :name: bulk_import :worker_name: BulkImportWorker :feature_category: :importers diff --git a/app/workers/concerns/gitlab/bitbucket_server_import/object_importer.rb b/app/workers/concerns/gitlab/bitbucket_server_import/object_importer.rb new file mode 100644 index 00000000000..b209719479b --- /dev/null +++ b/app/workers/concerns/gitlab/bitbucket_server_import/object_importer.rb @@ -0,0 +1,99 @@ +# frozen_string_literal: true + +module Gitlab + module BitbucketServerImport + # ObjectImporter defines the base behaviour for every Sidekiq worker that + # imports a single resource such as a note or pull request. + module ObjectImporter + extend ActiveSupport::Concern + + included do + include ApplicationWorker + + data_consistency :always + + feature_category :importers + + worker_has_external_dependencies! + + sidekiq_retries_exhausted do |msg| + args = msg['args'] + jid = msg['jid'] + + # If a job is being exhausted we still want to notify the + # Gitlab::Import::AdvanceStageWorker to prevent the entire import from getting stuck + if args.length == 3 && (key = args.last) && key.is_a?(String) + JobWaiter.notify(key, jid) + end + end + end + + def perform(project_id, hash, notify_key) + project = Project.find_by_id(project_id) + + return unless project + + if project.import_state&.canceled? + info(project.id, message: 'project import canceled') + return + end + + import(project, hash) + ensure + notify_waiter(notify_key) + end + + private + + # project - An instance of `Project` to import the data into. + # hash - A Hash containing the details of the object to import. + def import(project, hash) + info(project.id, message: 'importer started') + + importer_class.new(project, hash).execute + + info(project.id, message: 'importer finished') + rescue ActiveRecord::RecordInvalid => e + # We do not raise exception to prevent job retry + track_exception(project, e) + rescue StandardError => e + track_and_raise_exception(project, e) + end + + def notify_waiter(key) + JobWaiter.notify(key, jid) + end + + # Returns the class to use for importing the object. + def importer_class + raise NotImplementedError + end + + def info(project_id, extra = {}) + Logger.info(log_attributes(project_id, extra)) + end + + def log_attributes(project_id, extra = {}) + extra.merge( + project_id: project_id, + importer: importer_class.name + ) + end + + def track_exception(project, exception, fail_import: false) + Gitlab::Import::ImportFailureService.track( + project_id: project.id, + error_source: importer_class.name, + exception: exception, + fail_import: fail_import + ) + end + + def track_and_raise_exception(project, exception, fail_import: false) + track_exception(project, exception, fail_import: fail_import) + + raise(exception) + end + end + end +end diff --git a/app/workers/concerns/gitlab/bitbucket_server_import/stage_methods.rb b/app/workers/concerns/gitlab/bitbucket_server_import/stage_methods.rb new file mode 100644 index 00000000000..db4e71051c0 --- /dev/null +++ b/app/workers/concerns/gitlab/bitbucket_server_import/stage_methods.rb @@ -0,0 +1,74 @@ +# frozen_string_literal: true + +module Gitlab + module BitbucketServerImport + module StageMethods + extend ActiveSupport::Concern + + included do + include ApplicationWorker + + worker_has_external_dependencies! + + feature_category :importers + + data_consistency :always + + sidekiq_options dead: false, retry: 3 + + sidekiq_retries_exhausted do |msg, e| + Gitlab::Import::ImportFailureService.track( + project_id: msg['args'][0], + exception: e, + fail_import: true + ) + end + end + + # project_id - The ID of the GitLab project to import the data into. + def perform(project_id) + info(project_id, message: 'starting stage') + + return unless (project = find_project(project_id)) + + import(project) + + info(project_id, message: 'stage finished') + rescue StandardError => e + Gitlab::Import::ImportFailureService.track( + project_id: project_id, + exception: e, + error_source: self.class.name, + fail_import: abort_on_failure + ) + + raise(e) + end + + def find_project(id) + # If the project has been marked as failed we want to bail out + # automatically. + # rubocop: disable CodeReuse/ActiveRecord + Project.joins_import_state.where(import_state: { status: :started }).find_by_id(id) + # rubocop: enable CodeReuse/ActiveRecord + end + + def abort_on_failure + false + end + + private + + def info(project_id, extra = {}) + Logger.info(log_attributes(project_id, extra)) + end + + def log_attributes(project_id, extra = {}) + extra.merge( + project_id: project_id, + import_stage: self.class.name + ) + end + end + end +end diff --git a/app/workers/gitlab/bitbucket_server_import/advance_stage_worker.rb b/app/workers/gitlab/bitbucket_server_import/advance_stage_worker.rb new file mode 100644 index 00000000000..2c8db639725 --- /dev/null +++ b/app/workers/gitlab/bitbucket_server_import/advance_stage_worker.rb @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +module Gitlab + module BitbucketServerImport + # AdvanceStageWorker is a worker used by the BitBucket Server Importer to wait for a + # number of jobs to complete, without blocking a thread. Once all jobs have + # been completed this worker will advance the import process to the next + # stage. + class AdvanceStageWorker # rubocop:disable Scalability/IdempotentWorker + include ApplicationWorker + include ::Gitlab::Import::AdvanceStage + + data_consistency :delayed + + sidekiq_options dead: false, retry: 3 + + feature_category :importers + + loggable_arguments 1, 2 + + # The known importer stages and their corresponding Sidekiq workers. + STAGES = { + notes: Stage::ImportNotesWorker, + lfs_objects: Stage::ImportLfsObjectsWorker, + finish: Stage::FinishImportWorker + }.freeze + + def find_import_state(project_id) + ProjectImportState.jid_by(project_id: project_id, status: :started) + end + + private + + def next_stage_worker(next_stage) + STAGES.fetch(next_stage.to_sym) + end + end + end +end diff --git a/app/workers/gitlab/bitbucket_server_import/import_lfs_object_worker.rb b/app/workers/gitlab/bitbucket_server_import/import_lfs_object_worker.rb new file mode 100644 index 00000000000..709a6018646 --- /dev/null +++ b/app/workers/gitlab/bitbucket_server_import/import_lfs_object_worker.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +module Gitlab + module BitbucketServerImport + class ImportLfsObjectWorker # rubocop:disable Scalability/IdempotentWorker + include ObjectImporter + + def importer_class + Importers::LfsObjectImporter + end + end + end +end diff --git a/app/workers/gitlab/bitbucket_server_import/import_pull_request_notes_worker.rb b/app/workers/gitlab/bitbucket_server_import/import_pull_request_notes_worker.rb new file mode 100644 index 00000000000..b06d6ccd132 --- /dev/null +++ b/app/workers/gitlab/bitbucket_server_import/import_pull_request_notes_worker.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +module Gitlab + module BitbucketServerImport + class ImportPullRequestNotesWorker # rubocop:disable Scalability/IdempotentWorker + include ObjectImporter + + def importer_class + Importers::PullRequestNotesImporter + end + end + end +end diff --git a/app/workers/gitlab/bitbucket_server_import/import_pull_request_worker.rb b/app/workers/gitlab/bitbucket_server_import/import_pull_request_worker.rb new file mode 100644 index 00000000000..7f70057d54f --- /dev/null +++ b/app/workers/gitlab/bitbucket_server_import/import_pull_request_worker.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +module Gitlab + module BitbucketServerImport + class ImportPullRequestWorker # rubocop:disable Scalability/IdempotentWorker + include ObjectImporter + + def importer_class + Importers::PullRequestImporter + end + end + end +end diff --git a/app/workers/gitlab/bitbucket_server_import/stage/finish_import_worker.rb b/app/workers/gitlab/bitbucket_server_import/stage/finish_import_worker.rb new file mode 100644 index 00000000000..e3c8508815f --- /dev/null +++ b/app/workers/gitlab/bitbucket_server_import/stage/finish_import_worker.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +module Gitlab + module BitbucketServerImport + module Stage + class FinishImportWorker # rubocop:disable Scalability/IdempotentWorker + include StageMethods + + private + + # project - An instance of Project. + def import(project) + project.after_import + + Gitlab::Import::Metrics.new(:bitbucket_server_importer, project).track_finished_import + end + end + end + end +end diff --git a/app/workers/gitlab/bitbucket_server_import/stage/import_lfs_objects_worker.rb b/app/workers/gitlab/bitbucket_server_import/stage/import_lfs_objects_worker.rb new file mode 100644 index 00000000000..1002047225c --- /dev/null +++ b/app/workers/gitlab/bitbucket_server_import/stage/import_lfs_objects_worker.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +module Gitlab + module BitbucketServerImport + module Stage + class ImportLfsObjectsWorker # rubocop:disable Scalability/IdempotentWorker + include StageMethods + + private + + # project - An instance of Project. + def import(project) + waiter = importer_class.new(project).execute + + project.import_state.refresh_jid_expiration + + AdvanceStageWorker.perform_async( + project.id, + { waiter.key => waiter.jobs_remaining }, + :finish + ) + end + + def importer_class + Importers::LfsObjectsImporter + end + end + end + end +end diff --git a/app/workers/gitlab/bitbucket_server_import/stage/import_notes_worker.rb b/app/workers/gitlab/bitbucket_server_import/stage/import_notes_worker.rb new file mode 100644 index 00000000000..b30f9305829 --- /dev/null +++ b/app/workers/gitlab/bitbucket_server_import/stage/import_notes_worker.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +module Gitlab + module BitbucketServerImport + module Stage + class ImportNotesWorker # rubocop:disable Scalability/IdempotentWorker + include StageMethods + + private + + # project - An instance of Project. + def import(project) + waiter = importer_class.new(project).execute + + project.import_state.refresh_jid_expiration + + AdvanceStageWorker.perform_async( + project.id, + { waiter.key => waiter.jobs_remaining }, + :lfs_objects + ) + end + + def importer_class + Importers::NotesImporter + end + end + end + end +end diff --git a/app/workers/gitlab/bitbucket_server_import/stage/import_pull_requests_worker.rb b/app/workers/gitlab/bitbucket_server_import/stage/import_pull_requests_worker.rb new file mode 100644 index 00000000000..9e3d570e20d --- /dev/null +++ b/app/workers/gitlab/bitbucket_server_import/stage/import_pull_requests_worker.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +module Gitlab + module BitbucketServerImport + module Stage + class ImportPullRequestsWorker # rubocop:disable Scalability/IdempotentWorker + include StageMethods + + private + + # project - An instance of Project. + def import(project) + waiter = importer_class.new(project).execute + + project.import_state.refresh_jid_expiration + + AdvanceStageWorker.perform_async( + project.id, + { waiter.key => waiter.jobs_remaining }, + :notes + ) + end + + def importer_class + Importers::PullRequestsImporter + end + end + end + end +end diff --git a/app/workers/gitlab/bitbucket_server_import/stage/import_repository_worker.rb b/app/workers/gitlab/bitbucket_server_import/stage/import_repository_worker.rb new file mode 100644 index 00000000000..b378d07d59c --- /dev/null +++ b/app/workers/gitlab/bitbucket_server_import/stage/import_repository_worker.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +module Gitlab + module BitbucketServerImport + module Stage + class ImportRepositoryWorker # rubocop:disable Scalability/IdempotentWorker + include StageMethods + + private + + # project - An instance of Project. + def import(project) + importer = importer_class.new(project) + + importer.execute + + ImportPullRequestsWorker.perform_async(project.id) + end + + def importer_class + Importers::RepositoryImporter + end + + def abort_on_failure + true + end + end + end + end +end |