Add latest changes from gitlab-org/gitlab@16-8-stable-eev16.8.0-rc42

11 files changed, 212 insertions, 12 deletions

diff --git a/lib/gitlab/github_import/importer/attachments/base_importer.rb b/lib/gitlab/github_import/importer/attachments/base_importer.rb
index eaff99aed43..844008f8087 100644
--- a/lib/gitlab/github_import/importer/attachments/base_importer.rb
+++ b/lib/gitlab/github_import/importer/attachments/base_importer.rb
@@ -16,9 +16,11 @@ module Gitlab
               batch.each do |record|
                 next if already_imported?(record)
 
-                Gitlab::GithubImport::ObjectCounter.increment(project, object_type, :fetched)
+                if has_attachments?(record)
+                  Gitlab::GithubImport::ObjectCounter.increment(project, object_type, :fetched)
 
-                yield record
+                  yield record
+                end
 
                 # We mark the object as imported immediately so we don't end up
                 # scheduling it multiple times.
@@ -48,6 +50,12 @@ module Gitlab
           def object_representation(object)
             representation_class.from_db_record(object)
           end
+
+          def has_attachments?(object)
+            return true if Feature.disabled?(:github_importer_attachments, project, type: :gitlab_com_derisk)
+
+            object_representation(object).has_attachments?
+          end
         end
       end
     end
diff --git a/lib/gitlab/github_import/importer/events/base_importer.rb b/lib/gitlab/github_import/importer/events/base_importer.rb
index 8218acf2bfb..1ebafec5afc 100644
--- a/lib/gitlab/github_import/importer/events/base_importer.rb
+++ b/lib/gitlab/github_import/importer/events/base_importer.rb
@@ -10,6 +10,7 @@ module Gitlab
           # client - An instance of `Gitlab::GithubImport::Client`.
           def initialize(project, client)
             @project = project
+            @client = client
             @user_finder = UserFinder.new(project, client)
           end
 
@@ -20,7 +21,7 @@ module Gitlab
 
           private
 
-          attr_reader :project, :user_finder
+          attr_reader :project, :user_finder, :client
 
           def author_id(issue_event, author_key: :actor)
             user_finder.author_id_for(issue_event, author_key: author_key).first
@@ -42,6 +43,10 @@ module Gitlab
             belongs_to_key = merge_request_event?(issue_event) ? :merge_request_id : :issue_id
             { belongs_to_key => issuable_db_id(issue_event) }
           end
+
+          def import_settings
+            @import_settings ||= Gitlab::GithubImport::Settings.new(project)
+          end
         end
       end
     end
diff --git a/lib/gitlab/github_import/importer/events/commented.rb b/lib/gitlab/github_import/importer/events/commented.rb
new file mode 100644
index 00000000000..c9ebc31fa06
--- /dev/null
+++ b/lib/gitlab/github_import/importer/events/commented.rb
@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module GithubImport
+    module Importer
+      module Events
+        class Commented < BaseImporter
+          def execute(issue_event)
+            return true unless import_settings.extended_events?
+
+            note = Representation::Note.from_json_hash(
+              noteable_id: issue_event.issuable_id,
+              noteable_type: issue_event.issuable_type,
+              author: issue_event.actor&.to_hash,
+              note: issue_event.body,
+              created_at: issue_event.created_at,
+              updated_at: issue_event.updated_at,
+              note_id: issue_event.id
+            )
+
+            NoteImporter.new(note, project, client).execute
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/github_import/importer/events/merged.rb b/lib/gitlab/github_import/importer/events/merged.rb
index 6189fa8f429..702ea7f1fd5 100644
--- a/lib/gitlab/github_import/importer/events/merged.rb
+++ b/lib/gitlab/github_import/importer/events/merged.rb
@@ -6,6 +6,8 @@ module Gitlab
       module Events
         class Merged < BaseImporter
           def execute(issue_event)
+            create_note(issue_event) if import_settings.extended_events?
+
             create_event(issue_event)
             create_state_event(issue_event)
           end
@@ -37,6 +39,17 @@ module Gitlab
 
             ResourceStateEvent.create!(attrs)
           end
+
+          def create_note(issue_event)
+            pull_request = Representation::PullRequest.from_json_hash({
+              merged_by: issue_event.actor&.to_hash,
+              merged_at: issue_event.created_at,
+              iid: issue_event.issuable_id,
+              state: :closed
+            })
+
+            PullRequests::MergedByImporter.new(pull_request, project, client).execute
+          end
         end
       end
     end
diff --git a/lib/gitlab/github_import/importer/events/reviewed.rb b/lib/gitlab/github_import/importer/events/reviewed.rb
new file mode 100644
index 00000000000..1c0e8a9e6e8
--- /dev/null
+++ b/lib/gitlab/github_import/importer/events/reviewed.rb
@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module GithubImport
+    module Importer
+      module Events
+        class Reviewed < BaseImporter
+          def execute(issue_event)
+            return true unless import_settings.extended_events?
+
+            review = Representation::PullRequestReview.new(
+              merge_request_iid: issue_event.issuable_id,
+              author: issue_event.actor&.to_hash,
+              note: issue_event.body.to_s,
+              review_type: issue_event.state.upcase, # On timeline API, the state is in lower case
+              submitted_at: issue_event.submitted_at,
+              review_id: issue_event.id
+            )
+
+            PullRequests::ReviewImporter.new(review, project, client).execute({ add_reviewer: false })
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/github_import/importer/issue_event_importer.rb b/lib/gitlab/github_import/importer/issue_event_importer.rb
index d20482eca6f..9f15e9a25d8 100644
--- a/lib/gitlab/github_import/importer/issue_event_importer.rb
+++ b/lib/gitlab/github_import/importer/issue_event_importer.rb
@@ -22,6 +22,17 @@ module Gitlab
           unlabeled
         ].freeze
 
+        EXTENDED_SUPPORTED_EVENTS = SUPPORTED_EVENTS + %w[
+          commented
+          reviewed
+        ].freeze
+
+        EVENT_COUNTER_MAP = {
+          'commented' => 'note',
+          'reviewed' => 'pull_request_review',
+          'merged' => 'pull_request_merged_by'
+        }.freeze
+
         # issue_event - An instance of `Gitlab::GithubImport::Representation::IssueEvent`.
         # project - An instance of `Project`.
         # client - An instance of `Gitlab::GithubImport::Client`.
@@ -65,6 +76,10 @@ module Gitlab
             Gitlab::GithubImport::Importer::Events::ChangedReviewer
           when 'merged'
             Gitlab::GithubImport::Importer::Events::Merged
+          when 'commented'
+            Gitlab::GithubImport::Importer::Events::Commented
+          when 'reviewed'
+            Gitlab::GithubImport::Importer::Events::Reviewed
           end
         end
       end
diff --git a/lib/gitlab/github_import/importer/note_attachments_importer.rb b/lib/gitlab/github_import/importer/note_attachments_importer.rb
index 26472b0d468..36a256bbef5 100644
--- a/lib/gitlab/github_import/importer/note_attachments_importer.rb
+++ b/lib/gitlab/github_import/importer/note_attachments_importer.rb
@@ -16,10 +16,9 @@ module Gitlab
         end
 
         def execute
-          attachments = MarkdownText.fetch_attachments(note_text.text)
-          return if attachments.blank?
+          return unless note_text.has_attachments?
 
-          new_text = attachments.reduce(note_text.text) do |text, attachment|
+          new_text = note_text.attachments.reduce(note_text.text) do |text, attachment|
             new_url = gitlab_attachment_link(attachment)
             text.gsub(attachment.url, new_url)
           end
diff --git a/lib/gitlab/github_import/importer/pull_requests/review_importer.rb b/lib/gitlab/github_import/importer/pull_requests/review_importer.rb
index 6df130eb6e8..384880651ef 100644
--- a/lib/gitlab/github_import/importer/pull_requests/review_importer.rb
+++ b/lib/gitlab/github_import/importer/pull_requests/review_importer.rb
@@ -14,10 +14,12 @@ module Gitlab
             @review = review
             @project = project
             @client = client
-            @merge_request = project.merge_requests.find_by_id(review.merge_request_id)
+            @merge_request = project.merge_requests.find_by_iid(review.merge_request_iid)
           end
 
-          def execute
+          def execute(options = {})
+            options = { add_reviewer: true }.merge(options)
+
             user_finder = GithubImport::UserFinder.new(project, client)
 
             gitlab_user_id = user_finder.user_id_for(review.author)
@@ -25,7 +27,7 @@ module Gitlab
             if gitlab_user_id
               add_review_note!(gitlab_user_id)
               add_approval!(gitlab_user_id)
-              add_reviewer!(gitlab_user_id)
+              add_reviewer!(gitlab_user_id) if options[:add_reviewer]
             else
               add_complementary_review_note!(project.creator_id)
             end
diff --git a/lib/gitlab/github_import/importer/pull_requests/reviews_importer.rb b/lib/gitlab/github_import/importer/pull_requests/reviews_importer.rb
index 347423b0e21..62c9e6469d7 100644
--- a/lib/gitlab/github_import/importer/pull_requests/reviews_importer.rb
+++ b/lib/gitlab/github_import/importer/pull_requests/reviews_importer.rb
@@ -72,7 +72,7 @@ module Gitlab
             merge_requests_to_import.find_each do |merge_request|
               # The page counter needs to be scoped by merge request to avoid skipping
               # pages of reviews from already imported merge requests.
-              page_counter = PageCounter.new(project, page_counter_id(merge_request))
+              page_counter = Gitlab::Import::PageCounter.new(project, page_counter_id(merge_request))
               repo = project.import_source
               options = collection_options.merge(page: page_counter.current)
 
diff --git a/lib/gitlab/github_import/importer/replay_events_importer.rb b/lib/gitlab/github_import/importer/replay_events_importer.rb
new file mode 100644
index 00000000000..83578cf7672
--- /dev/null
+++ b/lib/gitlab/github_import/importer/replay_events_importer.rb
@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module GithubImport
+    module Importer
+      class ReplayEventsImporter
+        SUPPORTED_EVENTS = %w[review_request_removed review_requested].freeze
+
+        # replay_event - An instance of `Gitlab::GithubImport::Representation::ReplayEvent`.
+        # project - An instance of `Project`
+        # client - An instance of `Gitlab::GithubImport::Client`
+        def initialize(replay_event, project, client)
+          @project = project
+          @client = client
+          @replay_event = replay_event
+        end
+
+        def execute
+          association = case replay_event.issuable_type
+                        when 'MergeRequest'
+                          project.merge_requests.find_by_iid(replay_event.issuable_iid)
+                        end
+
+          return unless association
+
+          events_cache = EventsCache.new(project)
+
+          handle_review_requests(association, events_cache.events(association))
+
+          events_cache.delete(association)
+        end
+
+        private
+
+        attr_reader :project, :client, :replay_event
+
+        def handle_review_requests(association, events)
+          reviewers = {}
+
+          events.each do |event|
+            case event.event
+            when 'review_requested'
+              reviewers[event.requested_reviewer.login] = event.requested_reviewer.to_hash if event.requested_reviewer
+            when 'review_request_removed'
+              reviewers[event.requested_reviewer.login] = nil if event.requested_reviewer
+            end
+          end
+
+          representation = Representation::PullRequests::ReviewRequests.from_json_hash(
+            merge_request_id: association.id,
+            merge_request_iid: association.iid,
+            users: reviewers.values.compact
+          )
+
+          Importer::PullRequests::ReviewRequestImporter.new(representation, project, client).execute
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/github_import/importer/single_endpoint_issue_events_importer.rb b/lib/gitlab/github_import/importer/single_endpoint_issue_events_importer.rb
index d7fa098a775..126a0b8fa4a 100644
--- a/lib/gitlab/github_import/importer/single_endpoint_issue_events_importer.rb
+++ b/lib/gitlab/github_import/importer/single_endpoint_issue_events_importer.rb
@@ -30,9 +30,11 @@ module Gitlab
 
           compose_associated_id!(parent_record, associated)
 
-          return if already_imported?(associated) || importer_class::SUPPORTED_EVENTS.exclude?(associated[:event])
+          return if already_imported?(associated) || supported_events.exclude?(associated[:event])
 
-          Gitlab::GithubImport::ObjectCounter.increment(project, object_type, :fetched)
+          cache_event(parent_record, associated)
+
+          increment_object_counter(associated[:event])
 
           pull_request = parent_record.is_a? MergeRequest
           associated[:issue] = { number: parent_record.iid, pull_request: pull_request }
@@ -64,6 +66,12 @@ module Gitlab
           :issue_event
         end
 
+        def increment_object_counter(event_name)
+          counter_type = importer_class::EVENT_COUNTER_MAP[event_name] if import_settings.extended_events?
+          counter_type ||= object_type
+          Gitlab::GithubImport::ObjectCounter.increment(project, counter_type, :fetched)
+        end
+
         def collection_method
           :issue_timeline
         end
@@ -98,6 +106,43 @@ module Gitlab
 
           event[:id] = "cross-reference##{issuable.iid}-in-#{event.dig(:source, :issue, :id)}"
         end
+
+        def import_settings
+          @import_settings ||= Gitlab::GithubImport::Settings.new(project)
+        end
+
+        def after_batch_processed(parent)
+          return unless import_settings.extended_events?
+
+          events = events_cache.events(parent)
+
+          return if events.empty?
+
+          hash = Representation::ReplayEvent.new(issuable_type: parent.class.name.to_s, issuable_iid: parent.iid)
+            .to_hash.deep_stringify_keys
+          ReplayEventsWorker.perform_async(project.id, hash, job_waiter.key.to_s)
+          job_waiter.jobs_remaining = Gitlab::Cache::Import::Caching.increment(job_waiter_remaining_cache_key)
+        end
+
+        def supported_events
+          return importer_class::EXTENDED_SUPPORTED_EVENTS if import_settings.extended_events?
+
+          importer_class::SUPPORTED_EVENTS
+        end
+
+        def cache_event(parent_record, associated)
+          return unless import_settings.extended_events?
+
+          return if Importer::ReplayEventsImporter::SUPPORTED_EVENTS.exclude?(associated[:event])
+
+          representation = representation_class.from_api_response(associated)
+
+          events_cache.add(parent_record, representation)
+        end
+
+        def events_cache
+          @events_cache ||= EventsCache.new(project)
+        end
       end
     end
   end