Add latest changes from gitlab-org/gitlab@16-4-stable-eev16.4.0-rc42

8 files changed, 212 insertions, 35 deletions

diff --git a/app/workers/concerns/gitlab/bitbucket_import/object_importer.rb b/app/workers/concerns/gitlab/bitbucket_import/object_importer.rb
new file mode 100644
index 00000000000..26e6e2675ed
--- /dev/null
+++ b/app/workers/concerns/gitlab/bitbucket_import/object_importer.rb
@@ -0,0 +1,98 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module BitbucketImport
+    # ObjectImporter defines the base behaviour for every Sidekiq worker that
+    # imports a single resource such as a note or pull request.
+    module ObjectImporter
+      extend ActiveSupport::Concern
+
+      included do
+        include ApplicationWorker
+
+        data_consistency :always
+
+        feature_category :importers
+
+        worker_has_external_dependencies!
+
+        sidekiq_retries_exhausted do |msg|
+          args = msg['args']
+          jid = msg['jid']
+
+          # If a job is being exhausted we still want to notify the
+          # Gitlab::Import::AdvanceStageWorker to prevent the entire import from getting stuck
+          key = args.last
+          JobWaiter.notify(key, jid) if args.length == 3 && key && key.is_a?(String)
+        end
+      end
+
+      def perform(project_id, hash, notify_key)
+        project = Project.find_by_id(project_id)
+
+        return unless project
+
+        if project.import_state&.canceled?
+          info(project.id, message: 'project import canceled')
+          return
+        end
+
+        import(project, hash)
+      ensure
+        notify_waiter(notify_key)
+      end
+
+      private
+
+      # project - An instance of `Project` to import the data into.
+      # hash - A Hash containing the details of the object to import.
+      def import(project, hash)
+        info(project.id, message: 'importer started')
+
+        importer_class.new(project, hash).execute
+
+        info(project.id, message: 'importer finished')
+      rescue ActiveRecord::RecordInvalid => e
+        # We do not raise exception to prevent job retry
+        track_exception(project, e)
+      rescue StandardError => e
+        track_and_raise_exception(project, e)
+      end
+
+      def notify_waiter(key)
+        JobWaiter.notify(key, jid)
+      end
+
+      # Returns the class to use for importing the object.
+      def importer_class
+        raise NotImplementedError
+      end
+
+      def info(project_id, extra = {})
+        Logger.info(log_attributes(project_id, extra))
+      end
+
+      def log_attributes(project_id, extra = {})
+        extra.merge(
+          project_id: project_id,
+          importer: importer_class.name
+        )
+      end
+
+      def track_exception(project, exception, fail_import: false)
+        Gitlab::Import::ImportFailureService.track(
+          project_id: project.id,
+          error_source: importer_class.name,
+          exception: exception,
+          fail_import: fail_import
+        )
+      end
+
+      def track_and_raise_exception(project, exception, fail_import: false)
+        track_exception(project, exception, fail_import: fail_import)
+
+        raise(exception)
+      end
+    end
+  end
+end
diff --git a/app/workers/concerns/gitlab/bitbucket_import/stage_methods.rb b/app/workers/concerns/gitlab/bitbucket_import/stage_methods.rb
new file mode 100644
index 00000000000..2885cc29532
--- /dev/null
+++ b/app/workers/concerns/gitlab/bitbucket_import/stage_methods.rb
@@ -0,0 +1,76 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module BitbucketImport
+    module StageMethods
+      extend ActiveSupport::Concern
+
+      included do
+        include ApplicationWorker
+
+        worker_has_external_dependencies!
+
+        feature_category :importers
+
+        data_consistency :always
+
+        sidekiq_options dead: false, retry: 3
+
+        sidekiq_retries_exhausted do |msg, e|
+          Gitlab::Import::ImportFailureService.track(
+            project_id: msg['args'][0],
+            exception: e,
+            fail_import: true
+          )
+        end
+      end
+
+      # project_id - The ID of the GitLab project to import the data into.
+      def perform(project_id)
+        info(project_id, message: 'starting stage')
+
+        project = find_project(project_id)
+
+        return unless project
+
+        import(project)
+
+        info(project_id, message: 'stage finished')
+      rescue StandardError => e
+        Gitlab::Import::ImportFailureService.track(
+          project_id: project_id,
+          exception: e,
+          error_source: self.class.name,
+          fail_import: abort_on_failure
+        )
+
+        raise(e)
+      end
+
+      def find_project(id)
+        # If the project has been marked as failed we want to bail out
+        # automatically.
+        # rubocop: disable CodeReuse/ActiveRecord
+        Project.joins_import_state.where(import_state: { status: :started }).find_by_id(id)
+        # rubocop: enable CodeReuse/ActiveRecord
+      end
+
+      def abort_on_failure
+        false
+      end
+
+      private
+
+      def info(project_id, extra = {})
+        Logger.info(log_attributes(project_id, extra))
+      end
+
+      def log_attributes(project_id, extra = {})
+        extra.merge(
+          project_id: project_id,
+          import_stage: self.class.name
+        )
+      end
+    end
+  end
+end
diff --git a/app/workers/concerns/gitlab/bitbucket_server_import/object_importer.rb b/app/workers/concerns/gitlab/bitbucket_server_import/object_importer.rb
index b209719479b..1090d82c922 100644
--- a/app/workers/concerns/gitlab/bitbucket_server_import/object_importer.rb
+++ b/app/workers/concerns/gitlab/bitbucket_server_import/object_importer.rb
@@ -23,7 +23,7 @@ module Gitlab
           # If a job is being exhausted we still want to notify the
           # Gitlab::Import::AdvanceStageWorker to prevent the entire import from getting stuck
           if args.length == 3 && (key = args.last) && key.is_a?(String)
-            JobWaiter.notify(key, jid)
+            JobWaiter.notify(key, jid, ttl: Gitlab::Import::JOB_WAITER_TTL)
           end
         end
       end
@@ -61,7 +61,7 @@ module Gitlab
       end
 
       def notify_waiter(key)
-        JobWaiter.notify(key, jid)
+        JobWaiter.notify(key, jid, ttl: Gitlab::Import::JOB_WAITER_TTL)
       end
 
       # Returns the class to use for importing the object.
diff --git a/app/workers/concerns/gitlab/github_import/object_importer.rb b/app/workers/concerns/gitlab/github_import/object_importer.rb
index 6cb9bd34969..e190ced5073 100644
--- a/app/workers/concerns/gitlab/github_import/object_importer.rb
+++ b/app/workers/concerns/gitlab/github_import/object_importer.rb
@@ -27,7 +27,7 @@ module Gitlab
           # If a job is being exhausted we still want to notify the
           # Gitlab::Import::AdvanceStageWorker to prevent the entire import from getting stuck
           if args.length == 3 && (key = args.last) && key.is_a?(String)
-            JobWaiter.notify(key, jid)
+            JobWaiter.notify(key, jid, ttl: Gitlab::Import::JOB_WAITER_TTL)
           end
         end
       end
@@ -38,7 +38,7 @@ module Gitlab
       # client - An instance of `Gitlab::GithubImport::Client`
       # hash - A Hash containing the details of the object to import.
       def import(project, client, hash)
-        unless project.import_state&.in_progress?
+        if project.import_state&.completed?
           info(
             project.id,
             message: 'Project import is no longer running. Stopping worker.',
diff --git a/app/workers/concerns/gitlab/github_import/rescheduling_methods.rb b/app/workers/concerns/gitlab/github_import/rescheduling_methods.rb
index b40914770b5..f6feb6d1598 100644
--- a/app/workers/concerns/gitlab/github_import/rescheduling_methods.rb
+++ b/app/workers/concerns/gitlab/github_import/rescheduling_methods.rb
@@ -39,7 +39,7 @@ module Gitlab
       end
 
       def notify_waiter(key = nil)
-        JobWaiter.notify(key, jid) if key
+        JobWaiter.notify(key, jid, ttl: Gitlab::Import::JOB_WAITER_TTL) if key
       end
 
       def reschedule_job(project, client, hash, notify_key)
diff --git a/app/workers/concerns/gitlab/github_import/stage_methods.rb b/app/workers/concerns/gitlab/github_import/stage_methods.rb
index a5287fcfbe2..75db5589415 100644
--- a/app/workers/concerns/gitlab/github_import/stage_methods.rb
+++ b/app/workers/concerns/gitlab/github_import/stage_methods.rb
@@ -9,7 +9,7 @@ module Gitlab
 
         return unless (project = find_project(project_id))
 
-        unless project.import_state&.in_progress?
+        if project.import_state&.completed?
           info(
             project_id,
             message: 'Project import is no longer running. Stopping worker.',
diff --git a/app/workers/concerns/gitlab/import/notify_upon_death.rb b/app/workers/concerns/gitlab/import/notify_upon_death.rb
new file mode 100644
index 00000000000..ae726a673c2
--- /dev/null
+++ b/app/workers/concerns/gitlab/import/notify_upon_death.rb
@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+# NotifyUponDeath can be included into a worker class if it should
+# notify any JobWaiter instances upon being moved to the Sidekiq dead queue.
+#
+# Note that this will only notify the waiter upon graceful termination, a
+# SIGKILL will still result in the waiter _not_ being notified.
+#
+# Workers including this module must have jobs passed where the last
+# argument is the key to notify, as a String.
+module Gitlab
+  module Import
+    module NotifyUponDeath
+      extend ActiveSupport::Concern
+
+      included do
+        # If a job is being exhausted we still want to notify the
+        # Gitlab::Import::AdvanceStageWorker. This prevents the entire import from getting stuck
+        # just because 1 job threw too many errors.
+        sidekiq_retries_exhausted do |job|
+          args = job['args']
+          jid = job['jid']
+          key = args.last
+
+          next unless args.length == 3 && key.is_a?(String)
+
+          JobWaiter.notify(key, jid, ttl: Gitlab::Import::JOB_WAITER_TTL)
+        end
+      end
+    end
+  end
+end
diff --git a/app/workers/concerns/gitlab/notify_upon_death.rb b/app/workers/concerns/gitlab/notify_upon_death.rb
deleted file mode 100644
index 66dc6270637..00000000000
--- a/app/workers/concerns/gitlab/notify_upon_death.rb
+++ /dev/null
@@ -1,29 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
-  # NotifyUponDeath can be included into a worker class if it should
-  # notify any JobWaiter instances upon being moved to the Sidekiq dead queue.
-  #
-  # Note that this will only notify the waiter upon graceful termination, a
-  # SIGKILL will still result in the waiter _not_ being notified.
-  #
-  # Workers including this module must have jobs passed where the last
-  # argument is the key to notify, as a String.
-  module NotifyUponDeath
-    extend ActiveSupport::Concern
-
-    included do
-      # If a job is being exhausted we still want to notify the
-      # Gitlab::Import::AdvanceStageWorker. This prevents the entire import from getting stuck
-      # just because 1 job threw too many errors.
-      sidekiq_retries_exhausted do |job|
-        args = job['args']
-        jid = job['jid']
-
-        if args.length == 3 && (key = args.last) && key.is_a?(String)
-          JobWaiter.notify(key, jid)
-        end
-      end
-    end
-  end
-end