diff options
Diffstat (limited to 'lib/gitlab/github_import')
11 files changed, 158 insertions, 15 deletions
diff --git a/lib/gitlab/github_import/parallel_importer.rb b/lib/gitlab/github_import/parallel_importer.rb index 2429fa4de1d..f72e595e8e9 100644 --- a/lib/gitlab/github_import/parallel_importer.rb +++ b/lib/gitlab/github_import/parallel_importer.rb @@ -15,6 +15,10 @@ module Gitlab true end + def self.track_start_import(project) + Gitlab::Import::Metrics.new(:github_importer, project).track_start_import + end + # This is a workaround for a Ruby 2.3.7 bug. rspec-mocks cannot restore # the visibility of prepended modules. See # https://github.com/rspec/rspec-mocks/issues/1231 for more details. diff --git a/lib/gitlab/github_import/parallel_scheduling.rb b/lib/gitlab/github_import/parallel_scheduling.rb index 4d0074e43d7..a8e006ea082 100644 --- a/lib/gitlab/github_import/parallel_scheduling.rb +++ b/lib/gitlab/github_import/parallel_scheduling.rb @@ -53,7 +53,8 @@ module Gitlab project_id: project.id, error_source: self.class.name, exception: e, - fail_import: abort_on_failure + fail_import: abort_on_failure, + metrics: true ) raise(e) diff --git a/lib/gitlab/github_import/representation/diff_note.rb b/lib/gitlab/github_import/representation/diff_note.rb index d0584cc6255..a3dcd2e380c 100644 --- a/lib/gitlab/github_import/representation/diff_note.rb +++ b/lib/gitlab/github_import/representation/diff_note.rb @@ -11,7 +11,7 @@ module Gitlab expose_attribute :noteable_type, :noteable_id, :commit_id, :file_path, :diff_hunk, :author, :note, :created_at, :updated_at, - :github_id, :original_commit_id + :original_commit_id, :note_id NOTEABLE_ID_REGEX = %r{/pull/(?<iid>\d+)}i.freeze @@ -40,7 +40,9 @@ module Gitlab note: note.body, created_at: note.created_at, updated_at: note.updated_at, - github_id: note.id + note_id: note.id, + end_line: note.line, + start_line: note.start_line } new(hash) @@ -82,6 +84,22 @@ module Gitlab new_file: false } end + + def note + @note ||= DiffNotes::SuggestionFormatter.formatted_note_for( + note: attributes[:note], + start_line: attributes[:start_line], + end_line: attributes[:end_line] + ) + end + + def github_identifiers + { + note_id: note_id, + noteable_id: noteable_id, + noteable_type: noteable_type + } + end end end end diff --git a/lib/gitlab/github_import/representation/diff_notes/suggestion_formatter.rb b/lib/gitlab/github_import/representation/diff_notes/suggestion_formatter.rb new file mode 100644 index 00000000000..4e5855ee4cd --- /dev/null +++ b/lib/gitlab/github_import/representation/diff_notes/suggestion_formatter.rb @@ -0,0 +1,66 @@ +# frozen_string_literal: true + +# This class replaces Github markdown suggestion tag with +# a Gitlab suggestion tag. The difference between +# Github's and Gitlab's suggestion tags is that Gitlab +# includes the range of the suggestion in the tag, while Github +# uses other note attributes to position the suggestion. +module Gitlab + module GithubImport + module Representation + module DiffNotes + class SuggestionFormatter + # A github suggestion: + # - the ```suggestion tag must be the first text of the line + # - it might have up to 3 spaces before the ```suggestion tag + # - extra text on the ```suggestion tag line will be ignored + GITHUB_SUGGESTION = /^\ {,3}(?<suggestion>```suggestion\b).*(?<eol>\R)/.freeze + + def self.formatted_note_for(...) + new(...).formatted_note + end + + def initialize(note:, start_line: nil, end_line: nil) + @note = note + @start_line = start_line + @end_line = end_line + end + + def formatted_note + if contains_suggestion? + note.gsub( + GITHUB_SUGGESTION, + "\\k<suggestion>:#{suggestion_range}\\k<eol>" + ) + else + note + end + end + + private + + attr_reader :note, :start_line, :end_line + + def contains_suggestion? + note.to_s.match?(GITHUB_SUGGESTION) + end + + # Github always saves the comment on the _last_ line of the range. + # Therefore, the diff hunk will always be related to lines before + # the comment itself. + def suggestion_range + "-#{line_count}+0" + end + + def line_count + if start_line.present? + end_line - start_line + else + 0 + end + end + end + end + end + end +end diff --git a/lib/gitlab/github_import/representation/issue.rb b/lib/gitlab/github_import/representation/issue.rb index 0e04b5ad57f..db4a8188c03 100644 --- a/lib/gitlab/github_import/representation/issue.rb +++ b/lib/gitlab/github_import/representation/issue.rb @@ -25,7 +25,6 @@ module Gitlab hash = { iid: issue.number, - github_id: issue.number, title: issue.title, description: issue.body, milestone_number: issue.milestone&.number, @@ -75,6 +74,13 @@ module Gitlab def issuable_type pull_request? ? 'MergeRequest' : 'Issue' end + + def github_identifiers + { + iid: iid, + issuable_type: issuable_type + } + end end end end diff --git a/lib/gitlab/github_import/representation/lfs_object.rb b/lib/gitlab/github_import/representation/lfs_object.rb index 41723759645..18737bfcde3 100644 --- a/lib/gitlab/github_import/representation/lfs_object.rb +++ b/lib/gitlab/github_import/representation/lfs_object.rb @@ -16,8 +16,7 @@ module Gitlab new( oid: lfs_object.oid, link: lfs_object.link, - size: lfs_object.size, - github_id: lfs_object.oid + size: lfs_object.size ) end @@ -31,6 +30,12 @@ module Gitlab def initialize(attributes) @attributes = attributes end + + def github_identifiers + { + oid: oid + } + end end end end diff --git a/lib/gitlab/github_import/representation/note.rb b/lib/gitlab/github_import/representation/note.rb index 5b98ce7d5ed..bcdb1a5459b 100644 --- a/lib/gitlab/github_import/representation/note.rb +++ b/lib/gitlab/github_import/representation/note.rb @@ -10,7 +10,7 @@ module Gitlab attr_reader :attributes expose_attribute :noteable_id, :noteable_type, :author, :note, - :created_at, :updated_at, :github_id + :created_at, :updated_at, :note_id NOTEABLE_TYPE_REGEX = %r{/(?<type>(pull|issues))/(?<iid>\d+)}i.freeze @@ -42,7 +42,7 @@ module Gitlab note: note.body, created_at: note.created_at, updated_at: note.updated_at, - github_id: note.id + note_id: note.id } new(hash) @@ -64,6 +64,14 @@ module Gitlab end alias_method :issuable_type, :noteable_type + + def github_identifiers + { + note_id: note_id, + noteable_id: noteable_id, + noteable_type: noteable_type + } + end end end end diff --git a/lib/gitlab/github_import/representation/pull_request.rb b/lib/gitlab/github_import/representation/pull_request.rb index e4f54fcc833..82bcdee8b2b 100644 --- a/lib/gitlab/github_import/representation/pull_request.rb +++ b/lib/gitlab/github_import/representation/pull_request.rb @@ -25,7 +25,6 @@ module Gitlab hash = { iid: pr.number, - github_id: pr.number, title: pr.title, description: pr.body, source_branch: pr.head.ref, @@ -108,6 +107,13 @@ module Gitlab def issuable_type 'MergeRequest' end + + def github_identifiers + { + iid: iid, + issuable_type: issuable_type + } + end end end end diff --git a/lib/gitlab/github_import/representation/pull_request_review.rb b/lib/gitlab/github_import/representation/pull_request_review.rb index 08b3160fc4c..70c1e51ffdd 100644 --- a/lib/gitlab/github_import/representation/pull_request_review.rb +++ b/lib/gitlab/github_import/representation/pull_request_review.rb @@ -9,7 +9,7 @@ module Gitlab attr_reader :attributes - expose_attribute :author, :note, :review_type, :submitted_at, :github_id, :merge_request_id + expose_attribute :author, :note, :review_type, :submitted_at, :merge_request_id, :review_id def self.from_api_response(review) user = Representation::User.from_api_response(review.user) if review.user @@ -20,7 +20,7 @@ module Gitlab note: review.body, review_type: review.state, submitted_at: review.submitted_at, - github_id: review.id + review_id: review.id ) end @@ -43,6 +43,13 @@ module Gitlab def approval? review_type == 'APPROVED' end + + def github_identifiers + { + review_id: review_id, + merge_request_id: merge_request_id + } + end end end end diff --git a/lib/gitlab/github_import/representation/user.rb b/lib/gitlab/github_import/representation/user.rb index d97b90b6291..fac8920a3f2 100644 --- a/lib/gitlab/github_import/representation/user.rb +++ b/lib/gitlab/github_import/representation/user.rb @@ -17,7 +17,6 @@ module Gitlab def self.from_api_response(user) new( id: user.id, - github_id: user.id, login: user.login ) end diff --git a/lib/gitlab/github_import/sequential_importer.rb b/lib/gitlab/github_import/sequential_importer.rb index cb6b2017208..6bc37337799 100644 --- a/lib/gitlab/github_import/sequential_importer.rb +++ b/lib/gitlab/github_import/sequential_importer.rb @@ -33,18 +33,41 @@ module Gitlab end def execute - Importer::RepositoryImporter.new(project, client).execute + metrics.track_start_import - SEQUENTIAL_IMPORTERS.each do |klass| - klass.new(project, client).execute + begin + Importer::RepositoryImporter.new(project, client).execute + + SEQUENTIAL_IMPORTERS.each do |klass| + klass.new(project, client).execute + end + + rescue StandardError => e + Gitlab::Import::ImportFailureService.track( + project_id: project.id, + error_source: self.class.name, + exception: e, + fail_import: true, + metrics: true + ) + + raise(e) end PARALLEL_IMPORTERS.each do |klass| klass.new(project, client, parallel: false).execute end + metrics.track_finished_import + true end + + private + + def metrics + @metrics ||= Gitlab::Import::Metrics.new(:github_importer, project) + end end end end |