diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2020-12-17 14:59:07 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2020-12-17 14:59:07 +0300 |
commit | 8b573c94895dc0ac0e1d9d59cf3e8745e8b539ca (patch) | |
tree | 544930fb309b30317ae9797a9683768705d664c4 /lib/gitlab/github_import | |
parent | 4b1de649d0168371549608993deac953eb692019 (diff) |
Add latest changes from gitlab-org/gitlab@13-7-stable-eev13.7.0-rc42
Diffstat (limited to 'lib/gitlab/github_import')
10 files changed, 335 insertions, 17 deletions
diff --git a/lib/gitlab/github_import/client.rb b/lib/gitlab/github_import/client.rb index dfe60fb5a03..328f1f742c5 100644 --- a/lib/gitlab/github_import/client.rb +++ b/lib/gitlab/github_import/client.rb @@ -69,6 +69,10 @@ module Gitlab with_rate_limit { octokit.user(username) } end + def pull_request_reviews(repo_name, iid) + with_rate_limit { octokit.pull_request_reviews(repo_name, iid) } + end + # Returns the details of a GitHub repository. # # name - The path (in the form `owner/repository`) of the repository. @@ -76,6 +80,10 @@ module Gitlab with_rate_limit { octokit.repo(name) } end + def pull_request(repo_name, iid) + with_rate_limit { octokit.pull_request(repo_name, iid) } + end + def labels(*args) each_object(:labels, *args) end @@ -155,8 +163,8 @@ module Gitlab end end - def search_repos_by_name(name) - each_page(:search_repositories, search_query(str: name, type: :name)) + def search_repos_by_name(name, options = {}) + octokit.search_repositories(search_query(str: name, type: :name), options) end def search_query(str:, type:, include_collaborations: true, include_orgs: true) diff --git a/lib/gitlab/github_import/importer/lfs_objects_importer.rb b/lib/gitlab/github_import/importer/lfs_objects_importer.rb index 5980b3c2179..c74a7706117 100644 --- a/lib/gitlab/github_import/importer/lfs_objects_importer.rb +++ b/lib/gitlab/github_import/importer/lfs_objects_importer.rb @@ -23,16 +23,13 @@ module Gitlab end def each_object_to_import - lfs_objects = Projects::LfsPointers::LfsImportService.new(project).execute + lfs_objects = Projects::LfsPointers::LfsObjectDownloadListService.new(project).execute lfs_objects.each do |object| yield object end rescue StandardError => e - Gitlab::Import::Logger.error( - message: 'The Lfs import process failed', - error: e.message - ) + error(project.id, e) end end end diff --git a/lib/gitlab/github_import/importer/pull_request_merged_by_importer.rb b/lib/gitlab/github_import/importer/pull_request_merged_by_importer.rb new file mode 100644 index 00000000000..11181edf0e9 --- /dev/null +++ b/lib/gitlab/github_import/importer/pull_request_merged_by_importer.rb @@ -0,0 +1,44 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Importer + class PullRequestMergedByImporter + def initialize(pull_request, project, client) + @project = project + @pull_request = pull_request + @client = client + end + + def execute + merge_request = project.merge_requests.find_by_iid(pull_request.iid) + user_finder = GithubImport::UserFinder.new(project, client) + gitlab_user_id = user_finder.user_id_for(pull_request.merged_by) + + if gitlab_user_id + timestamp = Time.new.utc + MergeRequest::Metrics.upsert({ + target_project_id: project.id, + merge_request_id: merge_request.id, + merged_by_id: gitlab_user_id, + created_at: timestamp, + updated_at: timestamp + }, unique_by: :merge_request_id) + else + merge_request.notes.create!( + importing: true, + note: "*Merged by: #{pull_request.merged_by.login}*", + author_id: project.creator_id, + project: project, + created_at: pull_request.created_at + ) + end + end + + private + + attr_reader :project, :pull_request, :client + end + end + end +end diff --git a/lib/gitlab/github_import/importer/pull_request_review_importer.rb b/lib/gitlab/github_import/importer/pull_request_review_importer.rb new file mode 100644 index 00000000000..14ee69ba089 --- /dev/null +++ b/lib/gitlab/github_import/importer/pull_request_review_importer.rb @@ -0,0 +1,101 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Importer + class PullRequestReviewImporter + def initialize(review, project, client) + @review = review + @project = project + @client = client + @merge_request = project.merge_requests.find_by_id(review.merge_request_id) + end + + def execute + user_finder = GithubImport::UserFinder.new(project, client) + gitlab_user_id = user_finder.user_id_for(review.author) + + if gitlab_user_id + add_review_note!(gitlab_user_id) + add_approval!(gitlab_user_id) + else + add_complementary_review_note!(project.creator_id) + end + end + + private + + attr_reader :review, :merge_request, :project, :client + + def add_review_note!(author_id) + return if review.note.empty? + + add_note!(author_id, review_note_content) + end + + def add_complementary_review_note!(author_id) + return if review.note.empty? && !review.approval? + + note = "*Created by %{login}*\n\n%{note}" % { + note: review_note_content, + login: review.author.login + } + + add_note!(author_id, note) + end + + def review_note_content + header = "**Review:** #{review.review_type.humanize}" + + if review.note.present? + "#{header}\n\n#{review.note}" + else + header + end + end + + def add_note!(author_id, note) + note = Note.new(note_attributes(author_id, note)) + + note.save! + end + + def note_attributes(author_id, note, extra = {}) + { + importing: true, + noteable_id: merge_request.id, + noteable_type: 'MergeRequest', + project_id: project.id, + author_id: author_id, + note: note, + system: false, + created_at: review.submitted_at, + updated_at: review.submitted_at + }.merge(extra) + end + + def add_approval!(user_id) + return unless review.review_type == 'APPROVED' + + add_approval_system_note!(user_id) + + merge_request.approvals.create!( + user_id: user_id, + created_at: review.submitted_at + ) + end + + def add_approval_system_note!(user_id) + attributes = note_attributes( + user_id, + 'approved this merge request', + system: true, + system_note_metadata: SystemNoteMetadata.new(action: 'approved') + ) + + Note.create!(attributes) + end + end + end + end +end diff --git a/lib/gitlab/github_import/importer/pull_requests_importer.rb b/lib/gitlab/github_import/importer/pull_requests_importer.rb index dcae8ca01fa..7f1569f592f 100644 --- a/lib/gitlab/github_import/importer/pull_requests_importer.rb +++ b/lib/gitlab/github_import/importer/pull_requests_importer.rb @@ -11,7 +11,7 @@ module Gitlab end def representation_class - Representation::PullRequest + Gitlab::GithubImport::Representation::PullRequest end def sidekiq_worker_class diff --git a/lib/gitlab/github_import/importer/pull_requests_merged_by_importer.rb b/lib/gitlab/github_import/importer/pull_requests_merged_by_importer.rb new file mode 100644 index 00000000000..466288fde4c --- /dev/null +++ b/lib/gitlab/github_import/importer/pull_requests_merged_by_importer.rb @@ -0,0 +1,38 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Importer + class PullRequestsMergedByImporter + include ParallelScheduling + + def importer_class + PullRequestMergedByImporter + end + + def representation_class + Gitlab::GithubImport::Representation::PullRequest + end + + def sidekiq_worker_class + ImportPullRequestMergedByWorker + end + + def collection_method + :pull_requests_merged_by + end + + def id_for_already_imported_cache(pr) + pr.number + end + + def each_object_to_import + project.merge_requests.with_state(:merged).find_each do |merge_request| + pull_request = client.pull_request(project.import_source, merge_request.iid) + yield(pull_request) + end + end + end + end + end +end diff --git a/lib/gitlab/github_import/importer/pull_requests_reviews_importer.rb b/lib/gitlab/github_import/importer/pull_requests_reviews_importer.rb new file mode 100644 index 00000000000..6d1b588f0e0 --- /dev/null +++ b/lib/gitlab/github_import/importer/pull_requests_reviews_importer.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Importer + class PullRequestsReviewsImporter + include ParallelScheduling + + def importer_class + PullRequestReviewImporter + end + + def representation_class + Gitlab::GithubImport::Representation::PullRequestReview + end + + def sidekiq_worker_class + ImportPullRequestReviewWorker + end + + def collection_method + :pull_request_reviews + end + + def id_for_already_imported_cache(review) + review.github_id + end + + def each_object_to_import + project.merge_requests.find_each do |merge_request| + reviews = client.pull_request_reviews(project.import_source, merge_request.iid) + reviews.each do |review| + review.merge_request_id = merge_request.id + yield(review) + end + end + end + end + end + end +end diff --git a/lib/gitlab/github_import/parallel_scheduling.rb b/lib/gitlab/github_import/parallel_scheduling.rb index cabc615ea11..51859010ec3 100644 --- a/lib/gitlab/github_import/parallel_scheduling.rb +++ b/lib/gitlab/github_import/parallel_scheduling.rb @@ -26,6 +26,8 @@ module Gitlab end def execute + info(project.id, message: "starting importer") + retval = if parallel? parallel_import @@ -43,8 +45,13 @@ module Gitlab # completed those jobs will just cycle through any remaining pages while # not scheduling anything. Gitlab::Cache::Import::Caching.expire(already_imported_cache_key, 15.minutes.to_i) + info(project.id, message: "importer finished") retval + rescue => e + error(project.id, e) + + raise e end # Imports all the objects in sequence in the current thread. @@ -157,6 +164,40 @@ module Gitlab def collection_options {} end + + private + + def info(project_id, extra = {}) + logger.info(log_attributes(project_id, extra)) + end + + def error(project_id, exception) + logger.error( + log_attributes( + project_id, + message: 'importer failed', + 'error.message': exception.message + ) + ) + + Gitlab::ErrorTracking.track_exception( + exception, + log_attributes(project_id) + ) + end + + def log_attributes(project_id, extra = {}) + extra.merge( + import_source: :github, + project_id: project_id, + importer: importer_class.name, + parallel: parallel? + ) + end + + def logger + @logger ||= Gitlab::Import::Logger.build + end end end end diff --git a/lib/gitlab/github_import/representation/pull_request.rb b/lib/gitlab/github_import/representation/pull_request.rb index 0ccc4bfaed3..be192762e05 100644 --- a/lib/gitlab/github_import/representation/pull_request.rb +++ b/lib/gitlab/github_import/representation/pull_request.rb @@ -13,18 +13,16 @@ module Gitlab :source_branch_sha, :target_branch, :target_branch_sha, :milestone_number, :author, :assignee, :created_at, :updated_at, :merged_at, :source_repository_id, - :target_repository_id, :source_repository_owner + :target_repository_id, :source_repository_owner, :merged_by # Builds a PR from a GitHub API response. # # issue - An instance of `Sawyer::Resource` containing the PR details. def self.from_api_response(pr) - assignee = - if pr.assignee - Representation::User.from_api_response(pr.assignee) - end - + assignee = Representation::User.from_api_response(pr.assignee) if pr.assignee user = Representation::User.from_api_response(pr.user) if pr.user + merged_by = Representation::User.from_api_response(pr.merged_by) if pr.merged_by + hash = { iid: pr.number, title: pr.title, @@ -42,7 +40,8 @@ module Gitlab assignee: assignee, created_at: pr.created_at, updated_at: pr.updated_at, - merged_at: pr.merged_at + merged_at: pr.merged_at, + merged_by: merged_by } new(hash) @@ -57,8 +56,8 @@ module Gitlab # Assignees are optional so we only convert it from a Hash if one was # set. - hash[:assignee] &&= Representation::User - .from_json_hash(hash[:assignee]) + hash[:assignee] &&= Representation::User.from_json_hash(hash[:assignee]) + hash[:merged_by] &&= Representation::User.from_json_hash(hash[:merged_by]) new(hash) end diff --git a/lib/gitlab/github_import/representation/pull_request_review.rb b/lib/gitlab/github_import/representation/pull_request_review.rb new file mode 100644 index 00000000000..3205259a1ed --- /dev/null +++ b/lib/gitlab/github_import/representation/pull_request_review.rb @@ -0,0 +1,49 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Representation + class PullRequestReview + include ToHash + include ExposeAttribute + + attr_reader :attributes + + expose_attribute :author, :note, :review_type, :submitted_at, :github_id, :merge_request_id + + def self.from_api_response(review) + user = Representation::User.from_api_response(review.user) if review.user + + new( + merge_request_id: review.merge_request_id, + author: user, + note: review.body, + review_type: review.state, + submitted_at: review.submitted_at, + github_id: review.id + ) + end + + # Builds a new note using a Hash that was built from a JSON payload. + def self.from_json_hash(raw_hash) + hash = Representation.symbolize_hash(raw_hash) + + hash[:author] &&= Representation::User.from_json_hash(hash[:author]) + hash[:submitted_at] = Time.parse(hash[:submitted_at]).in_time_zone + + new(hash) + end + + # attributes - A Hash containing the raw note details. The keys of this + # Hash must be Symbols. + def initialize(attributes) + @attributes = attributes + end + + def approval? + review_type == 'APPROVED' + end + end + end + end +end |