1 files changed, 29 insertions, 0 deletions
diff --git a/lib/gitlab/jira_import/issues_importer.rb b/lib/gitlab/jira_import/issues_importer.rb
index f1ead57c911..25dffcbe0ee 100644
--- a/lib/gitlab/jira_import/issues_importer.rb
+++ b/lib/gitlab/jira_import/issues_importer.rb
@@ -7,6 +7,10 @@ module Gitlab
       # see https://jira.atlassian.com/browse/JRACLOUD-67570
       # We set it to 1000 in case they change their mind.
       BATCH_SIZE = 1000
+      JIRA_IMPORT_THRESHOLD = 100_000
+      JIRA_IMPORT_PAUSE_LIMIT = 50_000
+
+      RetriesExceededError = Class.new(RuntimeError)
 
       attr_reader :imported_items_cache_key, :start_at, :job_waiter
 
@@ -66,6 +70,10 @@ module Gitlab
               @issue_type_id,
               { iid: next_iid }
             ).execute
+
+            # Pause the importer to allow the import to catch up and cache to drain
+            pause_jira_issue_importer if jira_import_issue_worker.queue_size > JIRA_IMPORT_THRESHOLD
+
             Gitlab::JiraImport::ImportIssueWorker.perform_async(project.id, jira_issue.id, issue_attrs, job_waiter.key)
 
             job_waiter.jobs_remaining += 1
@@ -89,6 +97,27 @@ module Gitlab
         job_waiter
       end
 
+      def jira_import_issue_worker
+        @_jira_import_issue_worker ||= Gitlab::JiraImport::ImportIssueWorker
+      end
+
+      def pause_jira_issue_importer
+        # Wait for import workers to drop below 50K in the iterations of the timeout
+        # timeout - Set to 5 seconds.
+        #           Time to process 100K jobs is currently ~14 seconds.
+        #           Source: https://github.com/mperham/sidekiq#performance
+        # retries - Set to 10 times to avoid indefinitely pause.
+        #           Raises an error if the queue does not reduce below the limit after 10 tries.
+
+        retries = 10
+        while retries > 0 && jira_import_issue_worker.queue_size >= JIRA_IMPORT_PAUSE_LIMIT
+          job_waiter.wait(5)
+          retries -= 1
+        end
+
+        raise RetriesExceededError, 'Retry failed after 10 attempts' if retries == 0
+      end
+
       def fetch_issues(start_at)
         client.Issue.jql("PROJECT='#{jira_project_key}' ORDER BY created ASC", { max_results: BATCH_SIZE, start_at: start_at })
       end