diff options
Diffstat (limited to 'lib/gitlab/github_import')
15 files changed, 56 insertions, 30 deletions
diff --git a/lib/gitlab/github_import/client.rb b/lib/gitlab/github_import/client.rb index 328f1f742c5..138716b1b53 100644 --- a/lib/gitlab/github_import/client.rb +++ b/lib/gitlab/github_import/client.rb @@ -70,7 +70,7 @@ module Gitlab end def pull_request_reviews(repo_name, iid) - with_rate_limit { octokit.pull_request_reviews(repo_name, iid) } + each_object(:pull_request_reviews, repo_name, iid) end # Returns the details of a GitHub repository. diff --git a/lib/gitlab/github_import/importer/diff_note_importer.rb b/lib/gitlab/github_import/importer/diff_note_importer.rb index 53b17f77ccd..d2f5af63621 100644 --- a/lib/gitlab/github_import/importer/diff_note_importer.rb +++ b/lib/gitlab/github_import/importer/diff_note_importer.rb @@ -21,8 +21,7 @@ module Gitlab author_id, author_found = user_finder.author_id_for(note) - note_body = - MarkdownText.format(note.note, note.author, author_found) + note_body = MarkdownText.format(note.note, note.author, author_found) attributes = { noteable_type: 'MergeRequest', diff --git a/lib/gitlab/github_import/importer/note_importer.rb b/lib/gitlab/github_import/importer/note_importer.rb index 41f179d275b..ae9996d81ef 100644 --- a/lib/gitlab/github_import/importer/note_importer.rb +++ b/lib/gitlab/github_import/importer/note_importer.rb @@ -21,8 +21,7 @@ module Gitlab author_id, author_found = user_finder.author_id_for(note) - note_body = - MarkdownText.format(note.note, note.author, author_found) + note_body = MarkdownText.format(note.note, note.author, author_found) attributes = { noteable_type: note.noteable_type, diff --git a/lib/gitlab/github_import/importer/pull_request_importer.rb b/lib/gitlab/github_import/importer/pull_request_importer.rb index f09e0bd9806..3c17ea1195e 100644 --- a/lib/gitlab/github_import/importer/pull_request_importer.rb +++ b/lib/gitlab/github_import/importer/pull_request_importer.rb @@ -44,8 +44,7 @@ module Gitlab def create_merge_request author_id, author_found = user_finder.author_id_for(pull_request) - description = MarkdownText - .format(pull_request.description, pull_request.author, author_found) + description = MarkdownText.format(pull_request.description, pull_request.author, author_found) attributes = { iid: pull_request.iid, diff --git a/lib/gitlab/github_import/importer/pull_request_review_importer.rb b/lib/gitlab/github_import/importer/pull_request_review_importer.rb index 9f495913897..f476ee13392 100644 --- a/lib/gitlab/github_import/importer/pull_request_review_importer.rb +++ b/lib/gitlab/github_import/importer/pull_request_review_importer.rb @@ -36,12 +36,12 @@ module Gitlab def add_complementary_review_note!(author_id) return if review.note.empty? && !review.approval? - note = "*Created by %{login}*\n\n%{note}" % { - note: review_note_content, - login: review.author.login - } + note_body = MarkdownText.format( + review_note_content, + review.author + ) - add_note!(author_id, note) + add_note!(author_id, note_body) end def review_note_content diff --git a/lib/gitlab/github_import/importer/pull_requests_merged_by_importer.rb b/lib/gitlab/github_import/importer/pull_requests_merged_by_importer.rb index 466288fde4c..94472cd341e 100644 --- a/lib/gitlab/github_import/importer/pull_requests_merged_by_importer.rb +++ b/lib/gitlab/github_import/importer/pull_requests_merged_by_importer.rb @@ -22,14 +22,18 @@ module Gitlab :pull_requests_merged_by end - def id_for_already_imported_cache(pr) - pr.number + def id_for_already_imported_cache(merge_request) + merge_request.id end def each_object_to_import project.merge_requests.with_state(:merged).find_each do |merge_request| + next if already_imported?(merge_request) + pull_request = client.pull_request(project.import_source, merge_request.iid) yield(pull_request) + + mark_as_imported(merge_request) end end end diff --git a/lib/gitlab/github_import/importer/pull_requests_reviews_importer.rb b/lib/gitlab/github_import/importer/pull_requests_reviews_importer.rb index 6d1b588f0e0..827027203ff 100644 --- a/lib/gitlab/github_import/importer/pull_requests_reviews_importer.rb +++ b/lib/gitlab/github_import/importer/pull_requests_reviews_importer.rb @@ -22,17 +22,22 @@ module Gitlab :pull_request_reviews end - def id_for_already_imported_cache(review) - review.github_id + def id_for_already_imported_cache(merge_request) + merge_request.id end def each_object_to_import project.merge_requests.find_each do |merge_request| - reviews = client.pull_request_reviews(project.import_source, merge_request.iid) - reviews.each do |review| - review.merge_request_id = merge_request.id - yield(review) - end + next if already_imported?(merge_request) + + client + .pull_request_reviews(project.import_source, merge_request.iid) + .each do |review| + review.merge_request_id = merge_request.id + yield(review) + end + + mark_as_imported(merge_request) end end end diff --git a/lib/gitlab/github_import/markdown_text.rb b/lib/gitlab/github_import/markdown_text.rb index b25c4f7becf..e5f4dabe42d 100644 --- a/lib/gitlab/github_import/markdown_text.rb +++ b/lib/gitlab/github_import/markdown_text.rb @@ -3,7 +3,7 @@ module Gitlab module GithubImport class MarkdownText - attr_reader :text, :author, :exists + include Gitlab::EncodingHelper def self.format(*args) new(*args).to_s @@ -19,10 +19,19 @@ module Gitlab end def to_s - if exists - text - else + # Gitlab::EncodingHelper#clean remove `null` chars from the string + clean(format) + end + + private + + attr_reader :text, :author, :exists + + def format + if author&.login.present? && !exists "*Created by: #{author.login}*\n\n#{text}" + else + text end end end diff --git a/lib/gitlab/github_import/parallel_importer.rb b/lib/gitlab/github_import/parallel_importer.rb index 1b4750da868..2429fa4de1d 100644 --- a/lib/gitlab/github_import/parallel_importer.rb +++ b/lib/gitlab/github_import/parallel_importer.rb @@ -40,4 +40,4 @@ module Gitlab end end -Gitlab::GithubImport::ParallelImporter.prepend_if_ee('::EE::Gitlab::GithubImport::ParallelImporter') +Gitlab::GithubImport::ParallelImporter.prepend_mod_with('Gitlab::GithubImport::ParallelImporter') diff --git a/lib/gitlab/github_import/parallel_scheduling.rb b/lib/gitlab/github_import/parallel_scheduling.rb index 51859010ec3..92f9e8a646d 100644 --- a/lib/gitlab/github_import/parallel_scheduling.rb +++ b/lib/gitlab/github_import/parallel_scheduling.rb @@ -48,7 +48,7 @@ module Gitlab info(project.id, message: "importer finished") retval - rescue => e + rescue StandardError => e error(project.id, e) raise e diff --git a/lib/gitlab/github_import/representation/issue.rb b/lib/gitlab/github_import/representation/issue.rb index f3071b3e2b3..0e04b5ad57f 100644 --- a/lib/gitlab/github_import/representation/issue.rb +++ b/lib/gitlab/github_import/representation/issue.rb @@ -25,6 +25,7 @@ module Gitlab hash = { iid: issue.number, + github_id: issue.number, title: issue.title, description: issue.body, milestone_number: issue.milestone&.number, diff --git a/lib/gitlab/github_import/representation/lfs_object.rb b/lib/gitlab/github_import/representation/lfs_object.rb index a4606173f49..41723759645 100644 --- a/lib/gitlab/github_import/representation/lfs_object.rb +++ b/lib/gitlab/github_import/representation/lfs_object.rb @@ -13,7 +13,12 @@ module Gitlab # Builds a lfs_object def self.from_api_response(lfs_object) - new({ oid: lfs_object.oid, link: lfs_object.link, size: lfs_object.size }) + new( + oid: lfs_object.oid, + link: lfs_object.link, + size: lfs_object.size, + github_id: lfs_object.oid + ) end # Builds a new lfs_object using a Hash that was built from a JSON payload. diff --git a/lib/gitlab/github_import/representation/pull_request.rb b/lib/gitlab/github_import/representation/pull_request.rb index be192762e05..e4f54fcc833 100644 --- a/lib/gitlab/github_import/representation/pull_request.rb +++ b/lib/gitlab/github_import/representation/pull_request.rb @@ -25,6 +25,7 @@ module Gitlab hash = { iid: pr.number, + github_id: pr.number, title: pr.title, description: pr.body, source_branch: pr.head.ref, diff --git a/lib/gitlab/github_import/representation/user.rb b/lib/gitlab/github_import/representation/user.rb index e00dcfca33d..d97b90b6291 100644 --- a/lib/gitlab/github_import/representation/user.rb +++ b/lib/gitlab/github_import/representation/user.rb @@ -15,7 +15,11 @@ module Gitlab # # user - An instance of `Sawyer::Resource` containing the user details. def self.from_api_response(user) - new(id: user.id, login: user.login) + new( + id: user.id, + github_id: user.id, + login: user.login + ) end # Builds a user using a Hash that was built from a JSON payload. diff --git a/lib/gitlab/github_import/user_finder.rb b/lib/gitlab/github_import/user_finder.rb index 34d1231b9a5..8d584415202 100644 --- a/lib/gitlab/github_import/user_finder.rb +++ b/lib/gitlab/github_import/user_finder.rb @@ -63,7 +63,7 @@ module Gitlab # # user - An instance of `Gitlab::GithubImport::Representation::User`. def user_id_for(user) - find(user.id, user.login) + find(user.id, user.login) if user.present? end # Returns the GitLab ID for the given GitHub ID or username. |