Add latest changes from gitlab-org/gitlab@16-0-stable-ee

12 files changed, 472 insertions, 0 deletions

diff --git a/app/workers/all_queues.yml b/app/workers/all_queues.yml
index 50b301b2fc3..17cc7bb73c2 100644
--- a/app/workers/all_queues.yml
+++ b/app/workers/all_queues.yml
@@ -2316,6 +2316,87 @@
   :weight: 1
   :idempotent: false
   :tags: []
+- :name: bitbucket_server_import_advance_stage
+  :worker_name: Gitlab::BitbucketServerImport::AdvanceStageWorker
+  :feature_category: :importers
+  :has_external_dependencies: false
+  :urgency: :low
+  :resource_boundary: :unknown
+  :weight: 1
+  :idempotent: false
+  :tags: []
+- :name: bitbucket_server_import_import_lfs_object
+  :worker_name: Gitlab::BitbucketServerImport::ImportLfsObjectWorker
+  :feature_category: :importers
+  :has_external_dependencies: true
+  :urgency: :low
+  :resource_boundary: :unknown
+  :weight: 1
+  :idempotent: false
+  :tags: []
+- :name: bitbucket_server_import_import_pull_request
+  :worker_name: Gitlab::BitbucketServerImport::ImportPullRequestWorker
+  :feature_category: :importers
+  :has_external_dependencies: true
+  :urgency: :low
+  :resource_boundary: :unknown
+  :weight: 1
+  :idempotent: false
+  :tags: []
+- :name: bitbucket_server_import_import_pull_request_notes
+  :worker_name: Gitlab::BitbucketServerImport::ImportPullRequestNotesWorker
+  :feature_category: :importers
+  :has_external_dependencies: true
+  :urgency: :low
+  :resource_boundary: :unknown
+  :weight: 1
+  :idempotent: false
+  :tags: []
+- :name: bitbucket_server_import_stage_finish_import
+  :worker_name: Gitlab::BitbucketServerImport::Stage::FinishImportWorker
+  :feature_category: :importers
+  :has_external_dependencies: true
+  :urgency: :low
+  :resource_boundary: :unknown
+  :weight: 1
+  :idempotent: false
+  :tags: []
+- :name: bitbucket_server_import_stage_import_lfs_objects
+  :worker_name: Gitlab::BitbucketServerImport::Stage::ImportLfsObjectsWorker
+  :feature_category: :importers
+  :has_external_dependencies: true
+  :urgency: :low
+  :resource_boundary: :unknown
+  :weight: 1
+  :idempotent: false
+  :tags: []
+- :name: bitbucket_server_import_stage_import_notes
+  :worker_name: Gitlab::BitbucketServerImport::Stage::ImportNotesWorker
+  :feature_category: :importers
+  :has_external_dependencies: true
+  :urgency: :low
+  :resource_boundary: :unknown
+  :weight: 1
+  :idempotent: false
+  :tags: []
+- :name: bitbucket_server_import_stage_import_pull_requests
+  :worker_name: Gitlab::BitbucketServerImport::Stage::ImportPullRequestsWorker
+  :feature_category: :importers
+  :has_external_dependencies: true
+  :urgency: :low
+  :resource_boundary: :unknown
+  :weight: 1
+  :idempotent: false
+  :tags: []
+- :name: bitbucket_server_import_stage_import_repository
+  :worker_name: Gitlab::BitbucketServerImport::Stage::ImportRepositoryWorker
+  :feature_category: :importers
+  :has_external_dependencies: true
+  :urgency: :low
+  :resource_boundary: :unknown
+  :weight: 1
+  :idempotent: false
+  :tags: []
 - :name: bulk_import
   :worker_name: BulkImportWorker
   :feature_category: :importers
diff --git a/app/workers/concerns/gitlab/bitbucket_server_import/object_importer.rb b/app/workers/concerns/gitlab/bitbucket_server_import/object_importer.rb
new file mode 100644
index 00000000000..b209719479b
--- /dev/null
+++ b/app/workers/concerns/gitlab/bitbucket_server_import/object_importer.rb
@@ -0,0 +1,99 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module BitbucketServerImport
+    # ObjectImporter defines the base behaviour for every Sidekiq worker that
+    # imports a single resource such as a note or pull request.
+    module ObjectImporter
+      extend ActiveSupport::Concern
+
+      included do
+        include ApplicationWorker
+
+        data_consistency :always
+
+        feature_category :importers
+
+        worker_has_external_dependencies!
+
+        sidekiq_retries_exhausted do |msg|
+          args = msg['args']
+          jid = msg['jid']
+
+          # If a job is being exhausted we still want to notify the
+          # Gitlab::Import::AdvanceStageWorker to prevent the entire import from getting stuck
+          if args.length == 3 && (key = args.last) && key.is_a?(String)
+            JobWaiter.notify(key, jid)
+          end
+        end
+      end
+
+      def perform(project_id, hash, notify_key)
+        project = Project.find_by_id(project_id)
+
+        return unless project
+
+        if project.import_state&.canceled?
+          info(project.id, message: 'project import canceled')
+          return
+        end
+
+        import(project, hash)
+      ensure
+        notify_waiter(notify_key)
+      end
+
+      private
+
+      # project - An instance of `Project` to import the data into.
+      # hash - A Hash containing the details of the object to import.
+      def import(project, hash)
+        info(project.id, message: 'importer started')
+
+        importer_class.new(project, hash).execute
+
+        info(project.id, message: 'importer finished')
+      rescue ActiveRecord::RecordInvalid => e
+        # We do not raise exception to prevent job retry
+        track_exception(project, e)
+      rescue StandardError => e
+        track_and_raise_exception(project, e)
+      end
+
+      def notify_waiter(key)
+        JobWaiter.notify(key, jid)
+      end
+
+      # Returns the class to use for importing the object.
+      def importer_class
+        raise NotImplementedError
+      end
+
+      def info(project_id, extra = {})
+        Logger.info(log_attributes(project_id, extra))
+      end
+
+      def log_attributes(project_id, extra = {})
+        extra.merge(
+          project_id: project_id,
+          importer: importer_class.name
+        )
+      end
+
+      def track_exception(project, exception, fail_import: false)
+        Gitlab::Import::ImportFailureService.track(
+          project_id: project.id,
+          error_source: importer_class.name,
+          exception: exception,
+          fail_import: fail_import
+        )
+      end
+
+      def track_and_raise_exception(project, exception, fail_import: false)
+        track_exception(project, exception, fail_import: fail_import)
+
+        raise(exception)
+      end
+    end
+  end
+end
diff --git a/app/workers/concerns/gitlab/bitbucket_server_import/stage_methods.rb b/app/workers/concerns/gitlab/bitbucket_server_import/stage_methods.rb
new file mode 100644
index 00000000000..db4e71051c0
--- /dev/null
+++ b/app/workers/concerns/gitlab/bitbucket_server_import/stage_methods.rb
@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module BitbucketServerImport
+    module StageMethods
+      extend ActiveSupport::Concern
+
+      included do
+        include ApplicationWorker
+
+        worker_has_external_dependencies!
+
+        feature_category :importers
+
+        data_consistency :always
+
+        sidekiq_options dead: false, retry: 3
+
+        sidekiq_retries_exhausted do |msg, e|
+          Gitlab::Import::ImportFailureService.track(
+            project_id: msg['args'][0],
+            exception: e,
+            fail_import: true
+          )
+        end
+      end
+
+      # project_id - The ID of the GitLab project to import the data into.
+      def perform(project_id)
+        info(project_id, message: 'starting stage')
+
+        return unless (project = find_project(project_id))
+
+        import(project)
+
+        info(project_id, message: 'stage finished')
+      rescue StandardError => e
+        Gitlab::Import::ImportFailureService.track(
+          project_id: project_id,
+          exception: e,
+          error_source: self.class.name,
+          fail_import: abort_on_failure
+        )
+
+        raise(e)
+      end
+
+      def find_project(id)
+        # If the project has been marked as failed we want to bail out
+        # automatically.
+        # rubocop: disable CodeReuse/ActiveRecord
+        Project.joins_import_state.where(import_state: { status: :started }).find_by_id(id)
+        # rubocop: enable CodeReuse/ActiveRecord
+      end
+
+      def abort_on_failure
+        false
+      end
+
+      private
+
+      def info(project_id, extra = {})
+        Logger.info(log_attributes(project_id, extra))
+      end
+
+      def log_attributes(project_id, extra = {})
+        extra.merge(
+          project_id: project_id,
+          import_stage: self.class.name
+        )
+      end
+    end
+  end
+end
diff --git a/app/workers/gitlab/bitbucket_server_import/advance_stage_worker.rb b/app/workers/gitlab/bitbucket_server_import/advance_stage_worker.rb
new file mode 100644
index 00000000000..2c8db639725
--- /dev/null
+++ b/app/workers/gitlab/bitbucket_server_import/advance_stage_worker.rb
@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module BitbucketServerImport
+    # AdvanceStageWorker is a worker used by the BitBucket Server Importer to wait for a
+    # number of jobs to complete, without blocking a thread. Once all jobs have
+    # been completed this worker will advance the import process to the next
+    # stage.
+    class AdvanceStageWorker # rubocop:disable Scalability/IdempotentWorker
+      include ApplicationWorker
+      include ::Gitlab::Import::AdvanceStage
+
+      data_consistency :delayed
+
+      sidekiq_options dead: false, retry: 3
+
+      feature_category :importers
+
+      loggable_arguments 1, 2
+
+      # The known importer stages and their corresponding Sidekiq workers.
+      STAGES = {
+        notes: Stage::ImportNotesWorker,
+        lfs_objects: Stage::ImportLfsObjectsWorker,
+        finish: Stage::FinishImportWorker
+      }.freeze
+
+      def find_import_state(project_id)
+        ProjectImportState.jid_by(project_id: project_id, status: :started)
+      end
+
+      private
+
+      def next_stage_worker(next_stage)
+        STAGES.fetch(next_stage.to_sym)
+      end
+    end
+  end
+end
diff --git a/app/workers/gitlab/bitbucket_server_import/import_lfs_object_worker.rb b/app/workers/gitlab/bitbucket_server_import/import_lfs_object_worker.rb
new file mode 100644
index 00000000000..709a6018646
--- /dev/null
+++ b/app/workers/gitlab/bitbucket_server_import/import_lfs_object_worker.rb
@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module BitbucketServerImport
+    class ImportLfsObjectWorker # rubocop:disable Scalability/IdempotentWorker
+      include ObjectImporter
+
+      def importer_class
+        Importers::LfsObjectImporter
+      end
+    end
+  end
+end
diff --git a/app/workers/gitlab/bitbucket_server_import/import_pull_request_notes_worker.rb b/app/workers/gitlab/bitbucket_server_import/import_pull_request_notes_worker.rb
new file mode 100644
index 00000000000..b06d6ccd132
--- /dev/null
+++ b/app/workers/gitlab/bitbucket_server_import/import_pull_request_notes_worker.rb
@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module BitbucketServerImport
+    class ImportPullRequestNotesWorker # rubocop:disable Scalability/IdempotentWorker
+      include ObjectImporter
+
+      def importer_class
+        Importers::PullRequestNotesImporter
+      end
+    end
+  end
+end
diff --git a/app/workers/gitlab/bitbucket_server_import/import_pull_request_worker.rb b/app/workers/gitlab/bitbucket_server_import/import_pull_request_worker.rb
new file mode 100644
index 00000000000..7f70057d54f
--- /dev/null
+++ b/app/workers/gitlab/bitbucket_server_import/import_pull_request_worker.rb
@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module BitbucketServerImport
+    class ImportPullRequestWorker # rubocop:disable Scalability/IdempotentWorker
+      include ObjectImporter
+
+      def importer_class
+        Importers::PullRequestImporter
+      end
+    end
+  end
+end
diff --git a/app/workers/gitlab/bitbucket_server_import/stage/finish_import_worker.rb b/app/workers/gitlab/bitbucket_server_import/stage/finish_import_worker.rb
new file mode 100644
index 00000000000..e3c8508815f
--- /dev/null
+++ b/app/workers/gitlab/bitbucket_server_import/stage/finish_import_worker.rb
@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module BitbucketServerImport
+    module Stage
+      class FinishImportWorker # rubocop:disable Scalability/IdempotentWorker
+        include StageMethods
+
+        private
+
+        # project - An instance of Project.
+        def import(project)
+          project.after_import
+
+          Gitlab::Import::Metrics.new(:bitbucket_server_importer, project).track_finished_import
+        end
+      end
+    end
+  end
+end
diff --git a/app/workers/gitlab/bitbucket_server_import/stage/import_lfs_objects_worker.rb b/app/workers/gitlab/bitbucket_server_import/stage/import_lfs_objects_worker.rb
new file mode 100644
index 00000000000..1002047225c
--- /dev/null
+++ b/app/workers/gitlab/bitbucket_server_import/stage/import_lfs_objects_worker.rb
@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module BitbucketServerImport
+    module Stage
+      class ImportLfsObjectsWorker # rubocop:disable Scalability/IdempotentWorker
+        include StageMethods
+
+        private
+
+        # project - An instance of Project.
+        def import(project)
+          waiter = importer_class.new(project).execute
+
+          project.import_state.refresh_jid_expiration
+
+          AdvanceStageWorker.perform_async(
+            project.id,
+            { waiter.key => waiter.jobs_remaining },
+            :finish
+          )
+        end
+
+        def importer_class
+          Importers::LfsObjectsImporter
+        end
+      end
+    end
+  end
+end
diff --git a/app/workers/gitlab/bitbucket_server_import/stage/import_notes_worker.rb b/app/workers/gitlab/bitbucket_server_import/stage/import_notes_worker.rb
new file mode 100644
index 00000000000..b30f9305829
--- /dev/null
+++ b/app/workers/gitlab/bitbucket_server_import/stage/import_notes_worker.rb
@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module BitbucketServerImport
+    module Stage
+      class ImportNotesWorker # rubocop:disable Scalability/IdempotentWorker
+        include StageMethods
+
+        private
+
+        # project - An instance of Project.
+        def import(project)
+          waiter = importer_class.new(project).execute
+
+          project.import_state.refresh_jid_expiration
+
+          AdvanceStageWorker.perform_async(
+            project.id,
+            { waiter.key => waiter.jobs_remaining },
+            :lfs_objects
+          )
+        end
+
+        def importer_class
+          Importers::NotesImporter
+        end
+      end
+    end
+  end
+end
diff --git a/app/workers/gitlab/bitbucket_server_import/stage/import_pull_requests_worker.rb b/app/workers/gitlab/bitbucket_server_import/stage/import_pull_requests_worker.rb
new file mode 100644
index 00000000000..9e3d570e20d
--- /dev/null
+++ b/app/workers/gitlab/bitbucket_server_import/stage/import_pull_requests_worker.rb
@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module BitbucketServerImport
+    module Stage
+      class ImportPullRequestsWorker # rubocop:disable Scalability/IdempotentWorker
+        include StageMethods
+
+        private
+
+        # project - An instance of Project.
+        def import(project)
+          waiter = importer_class.new(project).execute
+
+          project.import_state.refresh_jid_expiration
+
+          AdvanceStageWorker.perform_async(
+            project.id,
+            { waiter.key => waiter.jobs_remaining },
+            :notes
+          )
+        end
+
+        def importer_class
+          Importers::PullRequestsImporter
+        end
+      end
+    end
+  end
+end
diff --git a/app/workers/gitlab/bitbucket_server_import/stage/import_repository_worker.rb b/app/workers/gitlab/bitbucket_server_import/stage/import_repository_worker.rb
new file mode 100644
index 00000000000..b378d07d59c
--- /dev/null
+++ b/app/workers/gitlab/bitbucket_server_import/stage/import_repository_worker.rb
@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module BitbucketServerImport
+    module Stage
+      class ImportRepositoryWorker # rubocop:disable Scalability/IdempotentWorker
+        include StageMethods
+
+        private
+
+        # project - An instance of Project.
+        def import(project)
+          importer = importer_class.new(project)
+
+          importer.execute
+
+          ImportPullRequestsWorker.perform_async(project.id)
+        end
+
+        def importer_class
+          Importers::RepositoryImporter
+        end
+
+        def abort_on_failure
+          true
+        end
+      end
+    end
+  end
+end