Add latest changes from gitlab-org/gitlab@master

3 files changed, 131 insertions, 0 deletions

diff --git a/lib/gitlab/jira_import/base_importer.rb b/lib/gitlab/jira_import/base_importer.rb
new file mode 100644
index 00000000000..24158b8e734
--- /dev/null
+++ b/lib/gitlab/jira_import/base_importer.rb
@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module JiraImport
+    class BaseImporter
+      attr_reader :project, :client, :formatter, :jira_project_key
+
+      def initialize(project)
+        raise Projects::ImportService::Error, _('Jira import feature is disabled.') unless Feature.enabled?(:jira_issue_import, project)
+        raise Projects::ImportService::Error, _('Jira integration not configured.') unless project.jira_service&.active?
+
+        @jira_project_key = project&.import_data&.becomes(JiraImportData)&.current_project&.key
+        raise Projects::ImportService::Error, _('Unable to find Jira project to import data from.') unless @jira_project_key
+
+        @project = project
+        @client = project.jira_service.client
+        @formatter = Gitlab::ImportFormatter.new
+      end
+
+      private
+
+      def imported_items_cache_key
+        raise NotImplementedError
+      end
+
+      def mark_as_imported(id)
+        Gitlab::Cache::Import::Caching.set_add(imported_items_cache_key, id)
+      end
+
+      def already_imported?(id)
+        Gitlab::Cache::Import::Caching.set_includes?(imported_items_cache_key, id)
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/jira_import/issue_serializer.rb b/lib/gitlab/jira_import/issue_serializer.rb
new file mode 100644
index 00000000000..f00852a21a2
--- /dev/null
+++ b/lib/gitlab/jira_import/issue_serializer.rb
@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module JiraImport
+    class IssueSerializer
+      def initialize(project, jira_issue, params = {})
+      end
+
+      def execute
+        # this is going to be implemented in https://gitlab.com/gitlab-org/gitlab/-/merge_requests/27201
+        {}
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/jira_import/issues_importer.rb b/lib/gitlab/jira_import/issues_importer.rb
new file mode 100644
index 00000000000..6543b633ddf
--- /dev/null
+++ b/lib/gitlab/jira_import/issues_importer.rb
@@ -0,0 +1,81 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module JiraImport
+    class IssuesImporter < BaseImporter
+      # Jira limits max items per request to be fetched to 100
+      # see https://jira.atlassian.com/browse/JRACLOUD-67570
+      # We set it to 1000 in case they change their mind.
+      BATCH_SIZE = 1000
+
+      attr_reader :imported_items_cache_key, :start_at, :job_waiter
+
+      def initialize(project)
+        super
+        # get cached start_at value, or zero if not cached yet
+        @start_at = Gitlab::JiraImport.get_issues_next_start_at(project.id)
+        @imported_items_cache_key = JiraImport.already_imported_cache_key(:issues, project.id)
+        @job_waiter = JobWaiter.new
+      end
+
+      def execute
+        import_issues
+      end
+
+      private
+
+      def import_issues
+        return job_waiter if jira_last_page_reached?
+
+        issues = fetch_issues(start_at)
+        update_start_at_with(issues)
+
+        schedule_issue_import_workers(issues)
+      end
+
+      def jira_last_page_reached?
+        start_at < 0
+      end
+
+      def update_start_at_with(issues)
+        @start_at += issues.size
+
+        # store -1 if this is the last page to be imported, so no more `ImportIssuesWorker` workers are scheduled
+        # from Gitlab::JiraImport::Stage::ImportIssuesWorker#perform
+        @start_at = -1 if issues.blank?
+        Gitlab::JiraImport.store_issues_next_started_at(project.id, start_at)
+      end
+
+      def schedule_issue_import_workers(issues)
+        next_iid = project.issues.maximum(:iid).to_i + 1
+
+        issues.each do |jira_issue|
+          # Technically it's possible that the same work is performed multiple
+          # times, as Sidekiq doesn't guarantee there will ever only be one
+          # instance of a job or if for some reason the paginated results
+          # returned from Jira include issues there were returned before.
+          # For such cases we exit early if issue was already imported.
+          next if already_imported?(jira_issue.id)
+
+          issue_attrs = IssueSerializer.new(project, jira_issue, { iid: next_iid }).execute
+          Gitlab::JiraImport::ImportIssueWorker.perform_async(project.id, jira_issue.id, issue_attrs, job_waiter.key)
+
+          job_waiter.jobs_remaining += 1
+          next_iid += 1
+
+          # Mark the issue as imported immediately so we don't end up
+          # importing it multiple times within same import.
+          # These ids are cleaned-up when import finishes.
+          # see Gitlab::JiraImport::Stage::FinishImportWorker
+          mark_as_imported(jira_issue.id)
+        end
+
+        job_waiter
+      end
+
+      def fetch_issues(start_at)
+        client.Issue.jql("PROJECT='#{jira_project_key}' ORDER BY created ASC", { max_results: BATCH_SIZE, start_at: start_at })
+      end
+    end
+  end
+end