Add latest changes from gitlab-org/gitlab@13-7-stable-eev13.7.0-rc42

10 files changed, 335 insertions, 17 deletions

diff --git a/lib/gitlab/github_import/client.rb b/lib/gitlab/github_import/client.rb
index dfe60fb5a03..328f1f742c5 100644
--- a/lib/gitlab/github_import/client.rb
+++ b/lib/gitlab/github_import/client.rb
@@ -69,6 +69,10 @@ module Gitlab
         with_rate_limit { octokit.user(username) }
       end
 
+      def pull_request_reviews(repo_name, iid)
+        with_rate_limit { octokit.pull_request_reviews(repo_name, iid) }
+      end
+
       # Returns the details of a GitHub repository.
       #
       # name - The path (in the form `owner/repository`) of the repository.
@@ -76,6 +80,10 @@ module Gitlab
         with_rate_limit { octokit.repo(name) }
       end
 
+      def pull_request(repo_name, iid)
+        with_rate_limit { octokit.pull_request(repo_name, iid) }
+      end
+
       def labels(*args)
         each_object(:labels, *args)
       end
@@ -155,8 +163,8 @@ module Gitlab
         end
       end
 
-      def search_repos_by_name(name)
-        each_page(:search_repositories, search_query(str: name, type: :name))
+      def search_repos_by_name(name, options = {})
+        octokit.search_repositories(search_query(str: name, type: :name), options)
       end
 
       def search_query(str:, type:, include_collaborations: true, include_orgs: true)
diff --git a/lib/gitlab/github_import/importer/lfs_objects_importer.rb b/lib/gitlab/github_import/importer/lfs_objects_importer.rb
index 5980b3c2179..c74a7706117 100644
--- a/lib/gitlab/github_import/importer/lfs_objects_importer.rb
+++ b/lib/gitlab/github_import/importer/lfs_objects_importer.rb
@@ -23,16 +23,13 @@ module Gitlab
         end
 
         def each_object_to_import
-          lfs_objects = Projects::LfsPointers::LfsImportService.new(project).execute
+          lfs_objects = Projects::LfsPointers::LfsObjectDownloadListService.new(project).execute
 
           lfs_objects.each do |object|
             yield object
           end
         rescue StandardError => e
-          Gitlab::Import::Logger.error(
-            message: 'The Lfs import process failed',
-            error: e.message
-          )
+          error(project.id, e)
         end
       end
     end
diff --git a/lib/gitlab/github_import/importer/pull_request_merged_by_importer.rb b/lib/gitlab/github_import/importer/pull_request_merged_by_importer.rb
new file mode 100644
index 00000000000..11181edf0e9
--- /dev/null
+++ b/lib/gitlab/github_import/importer/pull_request_merged_by_importer.rb
@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module GithubImport
+    module Importer
+      class PullRequestMergedByImporter
+        def initialize(pull_request, project, client)
+          @project = project
+          @pull_request = pull_request
+          @client = client
+        end
+
+        def execute
+          merge_request = project.merge_requests.find_by_iid(pull_request.iid)
+          user_finder = GithubImport::UserFinder.new(project, client)
+          gitlab_user_id = user_finder.user_id_for(pull_request.merged_by)
+
+          if gitlab_user_id
+            timestamp = Time.new.utc
+            MergeRequest::Metrics.upsert({
+              target_project_id: project.id,
+              merge_request_id: merge_request.id,
+              merged_by_id: gitlab_user_id,
+              created_at: timestamp,
+              updated_at: timestamp
+            }, unique_by: :merge_request_id)
+          else
+            merge_request.notes.create!(
+              importing: true,
+              note: "*Merged by: #{pull_request.merged_by.login}*",
+              author_id: project.creator_id,
+              project: project,
+              created_at: pull_request.created_at
+            )
+          end
+        end
+
+        private
+
+        attr_reader :project, :pull_request, :client
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/github_import/importer/pull_request_review_importer.rb b/lib/gitlab/github_import/importer/pull_request_review_importer.rb
new file mode 100644
index 00000000000..14ee69ba089
--- /dev/null
+++ b/lib/gitlab/github_import/importer/pull_request_review_importer.rb
@@ -0,0 +1,101 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module GithubImport
+    module Importer
+      class PullRequestReviewImporter
+        def initialize(review, project, client)
+          @review = review
+          @project = project
+          @client = client
+          @merge_request = project.merge_requests.find_by_id(review.merge_request_id)
+        end
+
+        def execute
+          user_finder = GithubImport::UserFinder.new(project, client)
+          gitlab_user_id = user_finder.user_id_for(review.author)
+
+          if gitlab_user_id
+            add_review_note!(gitlab_user_id)
+            add_approval!(gitlab_user_id)
+          else
+            add_complementary_review_note!(project.creator_id)
+          end
+        end
+
+        private
+
+        attr_reader :review, :merge_request, :project, :client
+
+        def add_review_note!(author_id)
+          return if review.note.empty?
+
+          add_note!(author_id, review_note_content)
+        end
+
+        def add_complementary_review_note!(author_id)
+          return if review.note.empty? && !review.approval?
+
+          note = "*Created by %{login}*\n\n%{note}" % {
+            note: review_note_content,
+            login: review.author.login
+          }
+
+          add_note!(author_id, note)
+        end
+
+        def review_note_content
+          header = "**Review:** #{review.review_type.humanize}"
+
+          if review.note.present?
+            "#{header}\n\n#{review.note}"
+          else
+            header
+          end
+        end
+
+        def add_note!(author_id, note)
+          note = Note.new(note_attributes(author_id, note))
+
+          note.save!
+        end
+
+        def note_attributes(author_id, note, extra = {})
+          {
+            importing: true,
+            noteable_id: merge_request.id,
+            noteable_type: 'MergeRequest',
+            project_id: project.id,
+            author_id: author_id,
+            note: note,
+            system: false,
+            created_at: review.submitted_at,
+            updated_at: review.submitted_at
+          }.merge(extra)
+        end
+
+        def add_approval!(user_id)
+          return unless review.review_type == 'APPROVED'
+
+          add_approval_system_note!(user_id)
+
+          merge_request.approvals.create!(
+            user_id: user_id,
+            created_at: review.submitted_at
+          )
+        end
+
+        def add_approval_system_note!(user_id)
+          attributes = note_attributes(
+            user_id,
+            'approved this merge request',
+            system: true,
+            system_note_metadata: SystemNoteMetadata.new(action: 'approved')
+          )
+
+          Note.create!(attributes)
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/github_import/importer/pull_requests_importer.rb b/lib/gitlab/github_import/importer/pull_requests_importer.rb
index dcae8ca01fa..7f1569f592f 100644
--- a/lib/gitlab/github_import/importer/pull_requests_importer.rb
+++ b/lib/gitlab/github_import/importer/pull_requests_importer.rb
@@ -11,7 +11,7 @@ module Gitlab
         end
 
         def representation_class
-          Representation::PullRequest
+          Gitlab::GithubImport::Representation::PullRequest
         end
 
         def sidekiq_worker_class
diff --git a/lib/gitlab/github_import/importer/pull_requests_merged_by_importer.rb b/lib/gitlab/github_import/importer/pull_requests_merged_by_importer.rb
new file mode 100644
index 00000000000..466288fde4c
--- /dev/null
+++ b/lib/gitlab/github_import/importer/pull_requests_merged_by_importer.rb
@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module GithubImport
+    module Importer
+      class PullRequestsMergedByImporter
+        include ParallelScheduling
+
+        def importer_class
+          PullRequestMergedByImporter
+        end
+
+        def representation_class
+          Gitlab::GithubImport::Representation::PullRequest
+        end
+
+        def sidekiq_worker_class
+          ImportPullRequestMergedByWorker
+        end
+
+        def collection_method
+          :pull_requests_merged_by
+        end
+
+        def id_for_already_imported_cache(pr)
+          pr.number
+        end
+
+        def each_object_to_import
+          project.merge_requests.with_state(:merged).find_each do |merge_request|
+            pull_request = client.pull_request(project.import_source, merge_request.iid)
+            yield(pull_request)
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/github_import/importer/pull_requests_reviews_importer.rb b/lib/gitlab/github_import/importer/pull_requests_reviews_importer.rb
new file mode 100644
index 00000000000..6d1b588f0e0
--- /dev/null
+++ b/lib/gitlab/github_import/importer/pull_requests_reviews_importer.rb
@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module GithubImport
+    module Importer
+      class PullRequestsReviewsImporter
+        include ParallelScheduling
+
+        def importer_class
+          PullRequestReviewImporter
+        end
+
+        def representation_class
+          Gitlab::GithubImport::Representation::PullRequestReview
+        end
+
+        def sidekiq_worker_class
+          ImportPullRequestReviewWorker
+        end
+
+        def collection_method
+          :pull_request_reviews
+        end
+
+        def id_for_already_imported_cache(review)
+          review.github_id
+        end
+
+        def each_object_to_import
+          project.merge_requests.find_each do |merge_request|
+            reviews = client.pull_request_reviews(project.import_source, merge_request.iid)
+            reviews.each do |review|
+              review.merge_request_id = merge_request.id
+              yield(review)
+            end
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/github_import/parallel_scheduling.rb b/lib/gitlab/github_import/parallel_scheduling.rb
index cabc615ea11..51859010ec3 100644
--- a/lib/gitlab/github_import/parallel_scheduling.rb
+++ b/lib/gitlab/github_import/parallel_scheduling.rb
@@ -26,6 +26,8 @@ module Gitlab
       end
 
       def execute
+        info(project.id, message: "starting importer")
+
         retval =
           if parallel?
             parallel_import
@@ -43,8 +45,13 @@ module Gitlab
         # completed those jobs will just cycle through any remaining pages while
         # not scheduling anything.
         Gitlab::Cache::Import::Caching.expire(already_imported_cache_key, 15.minutes.to_i)
+        info(project.id, message: "importer finished")
 
         retval
+      rescue => e
+        error(project.id, e)
+
+        raise e
       end
 
       # Imports all the objects in sequence in the current thread.
@@ -157,6 +164,40 @@ module Gitlab
       def collection_options
         {}
       end
+
+      private
+
+      def info(project_id, extra = {})
+        logger.info(log_attributes(project_id, extra))
+      end
+
+      def error(project_id, exception)
+        logger.error(
+          log_attributes(
+            project_id,
+            message: 'importer failed',
+            'error.message': exception.message
+          )
+        )
+
+        Gitlab::ErrorTracking.track_exception(
+          exception,
+          log_attributes(project_id)
+        )
+      end
+
+      def log_attributes(project_id, extra = {})
+        extra.merge(
+          import_source: :github,
+          project_id: project_id,
+          importer: importer_class.name,
+          parallel: parallel?
+        )
+      end
+
+      def logger
+        @logger ||= Gitlab::Import::Logger.build
+      end
     end
   end
 end
diff --git a/lib/gitlab/github_import/representation/pull_request.rb b/lib/gitlab/github_import/representation/pull_request.rb
index 0ccc4bfaed3..be192762e05 100644
--- a/lib/gitlab/github_import/representation/pull_request.rb
+++ b/lib/gitlab/github_import/representation/pull_request.rb
@@ -13,18 +13,16 @@ module Gitlab
                          :source_branch_sha, :target_branch, :target_branch_sha,
                          :milestone_number, :author, :assignee, :created_at,
                          :updated_at, :merged_at, :source_repository_id,
-                         :target_repository_id, :source_repository_owner
+                         :target_repository_id, :source_repository_owner, :merged_by
 
         # Builds a PR from a GitHub API response.
         #
         # issue - An instance of `Sawyer::Resource` containing the PR details.
         def self.from_api_response(pr)
-          assignee =
-            if pr.assignee
-              Representation::User.from_api_response(pr.assignee)
-            end
-
+          assignee = Representation::User.from_api_response(pr.assignee) if pr.assignee
           user = Representation::User.from_api_response(pr.user) if pr.user
+          merged_by = Representation::User.from_api_response(pr.merged_by) if pr.merged_by
+
           hash = {
             iid: pr.number,
             title: pr.title,
@@ -42,7 +40,8 @@ module Gitlab
             assignee: assignee,
             created_at: pr.created_at,
             updated_at: pr.updated_at,
-            merged_at: pr.merged_at
+            merged_at: pr.merged_at,
+            merged_by: merged_by
           }
 
           new(hash)
@@ -57,8 +56,8 @@ module Gitlab
 
           # Assignees are optional so we only convert it from a Hash if one was
           # set.
-          hash[:assignee] &&= Representation::User
-            .from_json_hash(hash[:assignee])
+          hash[:assignee] &&= Representation::User.from_json_hash(hash[:assignee])
+          hash[:merged_by] &&= Representation::User.from_json_hash(hash[:merged_by])
 
           new(hash)
         end
diff --git a/lib/gitlab/github_import/representation/pull_request_review.rb b/lib/gitlab/github_import/representation/pull_request_review.rb
new file mode 100644
index 00000000000..3205259a1ed
--- /dev/null
+++ b/lib/gitlab/github_import/representation/pull_request_review.rb
@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module GithubImport
+    module Representation
+      class PullRequestReview
+        include ToHash
+        include ExposeAttribute
+
+        attr_reader :attributes
+
+        expose_attribute :author, :note, :review_type, :submitted_at, :github_id, :merge_request_id
+
+        def self.from_api_response(review)
+          user = Representation::User.from_api_response(review.user) if review.user
+
+          new(
+            merge_request_id: review.merge_request_id,
+            author: user,
+            note: review.body,
+            review_type: review.state,
+            submitted_at: review.submitted_at,
+            github_id: review.id
+          )
+        end
+
+        # Builds a new note using a Hash that was built from a JSON payload.
+        def self.from_json_hash(raw_hash)
+          hash = Representation.symbolize_hash(raw_hash)
+
+          hash[:author] &&= Representation::User.from_json_hash(hash[:author])
+          hash[:submitted_at] = Time.parse(hash[:submitted_at]).in_time_zone
+
+          new(hash)
+        end
+
+        # attributes - A Hash containing the raw note details. The keys of this
+        #              Hash must be Symbols.
+        def initialize(attributes)
+          @attributes = attributes
+        end
+
+        def approval?
+          review_type == 'APPROVED'
+        end
+      end
+    end
+  end
+end