diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2022-07-20 18:40:28 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2022-07-20 18:40:28 +0300 |
commit | b595cb0c1dec83de5bdee18284abe86614bed33b (patch) | |
tree | 8c3d4540f193c5ff98019352f554e921b3a41a72 /lib/gitlab/github_import | |
parent | 2f9104a328fc8a4bddeaa4627b595166d24671d0 (diff) |
Add latest changes from gitlab-org/gitlab@15-2-stable-eev15.2.0-rc42
Diffstat (limited to 'lib/gitlab/github_import')
16 files changed, 572 insertions, 60 deletions
diff --git a/lib/gitlab/github_import/importer/events/changed_label.rb b/lib/gitlab/github_import/importer/events/changed_label.rb new file mode 100644 index 00000000000..6c408158b02 --- /dev/null +++ b/lib/gitlab/github_import/importer/events/changed_label.rb @@ -0,0 +1,43 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Importer + module Events + class ChangedLabel + def initialize(project, user_id) + @project = project + @user_id = user_id + end + + # issue_event - An instance of `Gitlab::GithubImport::Representation::IssueEvent`. + def execute(issue_event) + create_event(issue_event) + end + + private + + attr_reader :project, :user_id + + def create_event(issue_event) + ResourceLabelEvent.create!( + issue_id: issue_event.issue_db_id, + user_id: user_id, + label_id: label_finder.id_for(issue_event.label_title), + action: action(issue_event.event), + created_at: issue_event.created_at + ) + end + + def label_finder + Gitlab::GithubImport::LabelFinder.new(project) + end + + def action(event_type) + event_type == 'unlabeled' ? 'remove' : 'add' + end + end + end + end + end +end diff --git a/lib/gitlab/github_import/importer/events/closed.rb b/lib/gitlab/github_import/importer/events/closed.rb new file mode 100644 index 00000000000..8b2136c9b24 --- /dev/null +++ b/lib/gitlab/github_import/importer/events/closed.rb @@ -0,0 +1,50 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Importer + module Events + class Closed + attr_reader :project, :user_id + + def initialize(project, user_id) + @project = project + @user_id = user_id + end + + # issue_event - An instance of `Gitlab::GithubImport::Representation::IssueEvent`. + def execute(issue_event) + create_event(issue_event) + create_state_event(issue_event) + end + + private + + def create_event(issue_event) + Event.create!( + project_id: project.id, + author_id: user_id, + action: 'closed', + target_type: Issue.name, + target_id: issue_event.issue_db_id, + created_at: issue_event.created_at, + updated_at: issue_event.created_at + ) + end + + def create_state_event(issue_event) + ResourceStateEvent.create!( + user_id: user_id, + issue_id: issue_event.issue_db_id, + source_commit: issue_event.commit_id, + state: 'closed', + close_after_error_tracking_resolve: false, + close_auto_resolve_prometheus_alert: false, + created_at: issue_event.created_at + ) + end + end + end + end + end +end diff --git a/lib/gitlab/github_import/importer/events/cross_referenced.rb b/lib/gitlab/github_import/importer/events/cross_referenced.rb new file mode 100644 index 00000000000..20b902cfe50 --- /dev/null +++ b/lib/gitlab/github_import/importer/events/cross_referenced.rb @@ -0,0 +1,86 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Importer + module Events + class CrossReferenced + attr_reader :project, :user_id + + def initialize(project, user_id) + @project = project + @user_id = user_id + end + + # issue_event - An instance of `Gitlab::GithubImport::Representation::IssueEvent`. + def execute(issue_event) + mentioned_in_record_class = mentioned_in_type(issue_event) + mentioned_in_number = issue_event.source.dig(:issue, :number) + mentioned_in_record = init_mentioned_in( + mentioned_in_record_class, mentioned_in_number + ) + return if mentioned_in_record.nil? + + note_body = cross_reference_note_content(mentioned_in_record.gfm_reference(project)) + track_activity(mentioned_in_record_class) + create_note(issue_event, note_body) + end + + private + + def track_activity(mentioned_in_class) + return if mentioned_in_class != Issue + + Gitlab::UsageDataCounters::HLLRedisCounter.track_event( + Gitlab::UsageDataCounters::IssueActivityUniqueCounter::ISSUE_CROSS_REFERENCED, + values: user_id + ) + end + + def create_note(issue_event, note_body) + Note.create!( + system: true, + noteable_type: Issue.name, + noteable_id: issue_event.issue_db_id, + project: project, + author_id: user_id, + note: note_body, + system_note_metadata: SystemNoteMetadata.new(action: 'cross_reference'), + created_at: issue_event.created_at + ) + end + + def mentioned_in_type(issue_event) + is_pull_request = issue_event.source.dig(:issue, :pull_request).present? + is_pull_request ? MergeRequest : Issue + end + + # record_class - Issue/MergeRequest + def init_mentioned_in(record_class, iid) + db_id = fetch_mentioned_in_db_id(record_class, iid) + return if db_id.nil? + + record = record_class.new(id: db_id, iid: iid) + record.project = project + record.readonly! + record + end + + # record_class - Issue/MergeRequest + def fetch_mentioned_in_db_id(record_class, number) + sawyer_mentioned_in_adapter = Struct.new(:iid, :issuable_type, keyword_init: true) + mentioned_in_adapter = sawyer_mentioned_in_adapter.new( + iid: number, issuable_type: record_class.name + ) + + Gitlab::GithubImport::IssuableFinder.new(project, mentioned_in_adapter).database_id + end + + def cross_reference_note_content(gfm_reference) + "#{::SystemNotes::IssuablesService.cross_reference_note_prefix}#{gfm_reference}" + end + end + end + end + end +end diff --git a/lib/gitlab/github_import/importer/events/renamed.rb b/lib/gitlab/github_import/importer/events/renamed.rb new file mode 100644 index 00000000000..6a11c492210 --- /dev/null +++ b/lib/gitlab/github_import/importer/events/renamed.rb @@ -0,0 +1,56 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Importer + module Events + class Renamed + def initialize(project, user_id) + @project = project + @user_id = user_id + end + + # issue_event - An instance of `Gitlab::GithubImport::Representation::IssueEvent` + def execute(issue_event) + Note.create!(note_params(issue_event)) + end + + private + + attr_reader :project, :user_id + + def note_params(issue_event) + { + noteable_id: issue_event.issue_db_id, + noteable_type: Issue.name, + project_id: project.id, + author_id: user_id, + note: parse_body(issue_event), + system: true, + created_at: issue_event.created_at, + updated_at: issue_event.created_at, + system_note_metadata: SystemNoteMetadata.new( + { + action: "title", + created_at: issue_event.created_at, + updated_at: issue_event.created_at + } + ) + } + end + + def parse_body(issue_event) + old_diffs, new_diffs = Gitlab::Diff::InlineDiff.new( + issue_event.old_title, issue_event.new_title + ).inline_diffs + + marked_old_title = Gitlab::Diff::InlineDiffMarkdownMarker.new(issue_event.old_title).mark(old_diffs) + marked_new_title = Gitlab::Diff::InlineDiffMarkdownMarker.new(issue_event.new_title).mark(new_diffs) + + "changed title from **#{marked_old_title}** to **#{marked_new_title}**" + end + end + end + end + end +end diff --git a/lib/gitlab/github_import/importer/events/reopened.rb b/lib/gitlab/github_import/importer/events/reopened.rb new file mode 100644 index 00000000000..c0f3802bc46 --- /dev/null +++ b/lib/gitlab/github_import/importer/events/reopened.rb @@ -0,0 +1,47 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Importer + module Events + class Reopened + attr_reader :project, :user_id + + def initialize(project, user_id) + @project = project + @user_id = user_id + end + + # issue_event - An instance of `Gitlab::GithubImport::Representation::IssueEvent`. + def execute(issue_event) + create_event(issue_event) + create_state_event(issue_event) + end + + private + + def create_event(issue_event) + Event.create!( + project_id: project.id, + author_id: user_id, + action: 'reopened', + target_type: Issue.name, + target_id: issue_event.issue_db_id, + created_at: issue_event.created_at, + updated_at: issue_event.created_at + ) + end + + def create_state_event(issue_event) + ResourceStateEvent.create!( + user_id: user_id, + issue_id: issue_event.issue_db_id, + state: 'reopened', + created_at: issue_event.created_at + ) + end + end + end + end + end +end diff --git a/lib/gitlab/github_import/importer/issue_event_importer.rb b/lib/gitlab/github_import/importer/issue_event_importer.rb new file mode 100644 index 00000000000..e451af61ec3 --- /dev/null +++ b/lib/gitlab/github_import/importer/issue_event_importer.rb @@ -0,0 +1,53 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Importer + class IssueEventImporter + attr_reader :issue_event, :project, :client, :user_finder + + # issue_event - An instance of `Gitlab::GithubImport::Representation::IssueEvent`. + # project - An instance of `Project`. + # client - An instance of `Gitlab::GithubImport::Client`. + def initialize(issue_event, project, client) + @issue_event = issue_event + @project = project + @client = client + @user_finder = UserFinder.new(project, client) + end + + def execute + case issue_event.event + when 'closed' + Gitlab::GithubImport::Importer::Events::Closed.new(project, author_id) + .execute(issue_event) + when 'reopened' + Gitlab::GithubImport::Importer::Events::Reopened.new(project, author_id) + .execute(issue_event) + when 'labeled', 'unlabeled' + Gitlab::GithubImport::Importer::Events::ChangedLabel.new(project, author_id) + .execute(issue_event) + when 'renamed' + Gitlab::GithubImport::Importer::Events::Renamed.new(project, author_id) + .execute(issue_event) + when 'cross-referenced' + Gitlab::GithubImport::Importer::Events::CrossReferenced.new(project, author_id) + .execute(issue_event) + else + Gitlab::GithubImport::Logger.debug( + message: 'UNSUPPORTED_EVENT_TYPE', + event_type: issue_event.event, event_github_id: issue_event.id + ) + end + end + + private + + def author_id + id, _status = user_finder.author_id_for(issue_event, author_key: :actor) + id + end + end + end + end +end diff --git a/lib/gitlab/github_import/importer/issue_importer.rb b/lib/gitlab/github_import/importer/issue_importer.rb index 35fd4bd88a0..e7d41856b04 100644 --- a/lib/gitlab/github_import/importer/issue_importer.rb +++ b/lib/gitlab/github_import/importer/issue_importer.rb @@ -51,6 +51,7 @@ module Gitlab title: issue.truncated_title, author_id: author_id, project_id: project.id, + namespace_id: project.project_namespace_id, description: description, milestone_id: milestone_finder.id_for(issue), state_id: ::Issue.available_states[issue.state], diff --git a/lib/gitlab/github_import/importer/note_importer.rb b/lib/gitlab/github_import/importer/note_importer.rb index 673f56b5753..1410006af26 100644 --- a/lib/gitlab/github_import/importer/note_importer.rb +++ b/lib/gitlab/github_import/importer/note_importer.rb @@ -21,14 +21,12 @@ module Gitlab author_id, author_found = user_finder.author_id_for(note) - note_body = MarkdownText.format(note.note, note.author, author_found) - attributes = { noteable_type: note.noteable_type, noteable_id: noteable_id, project_id: project.id, author_id: author_id, - note: note_body, + note: note_body(author_found), discussion_id: note.discussion_id, system: false, created_at: note.created_at, @@ -48,6 +46,13 @@ module Gitlab def find_noteable_id GithubImport::IssuableFinder.new(project, note).database_id end + + private + + def note_body(author_found) + text = MarkdownText.convert_ref_links(note.note, project) + MarkdownText.format(text, note.author, author_found) + end end end end diff --git a/lib/gitlab/github_import/importer/single_endpoint_diff_notes_importer.rb b/lib/gitlab/github_import/importer/single_endpoint_diff_notes_importer.rb index a2c3d1bd057..e1d9ae44065 100644 --- a/lib/gitlab/github_import/importer/single_endpoint_diff_notes_importer.rb +++ b/lib/gitlab/github_import/importer/single_endpoint_diff_notes_importer.rb @@ -37,15 +37,15 @@ module Gitlab private - def noteables - project.merge_requests.where.not(iid: already_imported_noteables) # rubocop: disable CodeReuse/ActiveRecord + def parent_collection + project.merge_requests.where.not(iid: already_imported_parents) # rubocop: disable CodeReuse/ActiveRecord end def page_counter_id(merge_request) "merge_request/#{merge_request.id}/#{collection_method}" end - def notes_imported_cache_key + def parent_imported_cache_key "github-importer/merge_request/diff_notes/already-imported/#{project.id}" end end diff --git a/lib/gitlab/github_import/importer/single_endpoint_issue_events_importer.rb b/lib/gitlab/github_import/importer/single_endpoint_issue_events_importer.rb new file mode 100644 index 00000000000..45bbc25e637 --- /dev/null +++ b/lib/gitlab/github_import/importer/single_endpoint_issue_events_importer.rb @@ -0,0 +1,82 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Importer + class SingleEndpointIssueEventsImporter + include ParallelScheduling + include SingleEndpointNotesImporting + + PROCESSED_PAGE_CACHE_KEY = 'issues/%{issue_iid}/%{collection}' + BATCH_SIZE = 100 + + def initialize(project, client, parallel: true) + @project = project + @client = client + @parallel = parallel + @already_imported_cache_key = ALREADY_IMPORTED_CACHE_KEY % + { project: project.id, collection: collection_method } + end + + def each_associated(parent_record, associated) + compose_associated_id!(parent_record, associated) + return if already_imported?(associated) + + Gitlab::GithubImport::ObjectCounter.increment(project, object_type, :fetched) + + associated.issue_db_id = parent_record.id + yield(associated) + + mark_as_imported(associated) + end + + def importer_class + IssueEventImporter + end + + def representation_class + Representation::IssueEvent + end + + def sidekiq_worker_class + ImportIssueEventWorker + end + + def object_type + :issue_event + end + + def collection_method + :issue_timeline + end + + def parent_collection + project.issues.where.not(iid: already_imported_parents).select(:id, :iid) # rubocop: disable CodeReuse/ActiveRecord + end + + def parent_imported_cache_key + "github-importer/issues/#{collection_method}/already-imported/#{project.id}" + end + + def page_counter_id(issue) + PROCESSED_PAGE_CACHE_KEY % { issue_iid: issue.iid, collection: collection_method } + end + + def id_for_already_imported_cache(event) + event.id + end + + def collection_options + { state: 'all', sort: 'created', direction: 'asc' } + end + + # Cross-referenced events on Github doesn't have id. + def compose_associated_id!(issue, event) + return if event.event != 'cross-referenced' + + event.id = "cross-reference##{issue.id}-in-#{event.source.issue.id}" + end + end + end + end +end diff --git a/lib/gitlab/github_import/importer/single_endpoint_issue_notes_importer.rb b/lib/gitlab/github_import/importer/single_endpoint_issue_notes_importer.rb index 49569ed52d8..fe64df45700 100644 --- a/lib/gitlab/github_import/importer/single_endpoint_issue_notes_importer.rb +++ b/lib/gitlab/github_import/importer/single_endpoint_issue_notes_importer.rb @@ -37,15 +37,15 @@ module Gitlab private - def noteables - project.issues.where.not(iid: already_imported_noteables) # rubocop: disable CodeReuse/ActiveRecord + def parent_collection + project.issues.where.not(iid: already_imported_parents) # rubocop: disable CodeReuse/ActiveRecord end def page_counter_id(issue) "issue/#{issue.id}/#{collection_method}" end - def notes_imported_cache_key + def parent_imported_cache_key "github-importer/issue/notes/already-imported/#{project.id}" end end diff --git a/lib/gitlab/github_import/importer/single_endpoint_merge_request_notes_importer.rb b/lib/gitlab/github_import/importer/single_endpoint_merge_request_notes_importer.rb index d837639c14d..3b1991d2b88 100644 --- a/lib/gitlab/github_import/importer/single_endpoint_merge_request_notes_importer.rb +++ b/lib/gitlab/github_import/importer/single_endpoint_merge_request_notes_importer.rb @@ -37,15 +37,15 @@ module Gitlab private - def noteables - project.merge_requests.where.not(iid: already_imported_noteables) # rubocop: disable CodeReuse/ActiveRecord + def parent_collection + project.merge_requests.where.not(iid: already_imported_parents) # rubocop: disable CodeReuse/ActiveRecord end def page_counter_id(merge_request) "merge_request/#{merge_request.id}/#{collection_method}" end - def notes_imported_cache_key + def parent_imported_cache_key "github-importer/merge_request/notes/already-imported/#{project.id}" end end diff --git a/lib/gitlab/github_import/markdown_text.rb b/lib/gitlab/github_import/markdown_text.rb index 0b1c221bbec..692016bd005 100644 --- a/lib/gitlab/github_import/markdown_text.rb +++ b/lib/gitlab/github_import/markdown_text.rb @@ -5,8 +5,34 @@ module Gitlab class MarkdownText include Gitlab::EncodingHelper - def self.format(*args) - new(*args).to_s + ISSUE_REF_MATCHER = '%{github_url}/%{import_source}/issues' + PULL_REF_MATCHER = '%{github_url}/%{import_source}/pull' + + class << self + def format(*args) + new(*args).to_s + end + + # Links like `https://domain.github.com/<namespace>/<project>/pull/<iid>` needs to be converted + def convert_ref_links(text, project) + matcher_options = { github_url: github_url, import_source: project.import_source } + issue_ref_matcher = ISSUE_REF_MATCHER % matcher_options + pull_ref_matcher = PULL_REF_MATCHER % matcher_options + + url_helpers = Rails.application.routes.url_helpers + text.gsub(issue_ref_matcher, url_helpers.project_issues_url(project)) + .gsub(pull_ref_matcher, url_helpers.project_merge_requests_url(project)) + end + + private + + # Returns github domain without slash in the end + def github_url + oauth_config = Gitlab::Auth::OAuth::Provider.config_for('github') || {} + url = oauth_config['url'].presence || 'https://github.com' + url = url.chop if url.end_with?('/') + url + end end # text - The Markdown text as a String. diff --git a/lib/gitlab/github_import/representation/issue_event.rb b/lib/gitlab/github_import/representation/issue_event.rb new file mode 100644 index 00000000000..9016338db3b --- /dev/null +++ b/lib/gitlab/github_import/representation/issue_event.rb @@ -0,0 +1,54 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Representation + class IssueEvent + include ToHash + include ExposeAttribute + + attr_reader :attributes + + expose_attribute :id, :actor, :event, :commit_id, :label_title, :old_title, :new_title, + :source, :created_at + expose_attribute :issue_db_id # set in SingleEndpointIssueEventsImporter#each_associated + + # Builds a event from a GitHub API response. + # + # event - An instance of `Sawyer::Resource` containing the event details. + def self.from_api_response(event) + new( + id: event.id, + actor: event.actor && Representation::User.from_api_response(event.actor), + event: event.event, + commit_id: event.commit_id, + label_title: event.label && event.label[:name], + old_title: event.rename && event.rename[:from], + new_title: event.rename && event.rename[:to], + source: event.source, + issue_db_id: event.issue_db_id, + created_at: event.created_at + ) + end + + # Builds a event using a Hash that was built from a JSON payload. + def self.from_json_hash(raw_hash) + hash = Representation.symbolize_hash(raw_hash) + hash[:actor] &&= Representation::User.from_json_hash(hash[:actor]) + + new(hash) + end + + # attributes - A Hash containing the event details. The keys of this + # Hash (and any nested hashes) must be symbols. + def initialize(attributes) + @attributes = attributes + end + + def github_identifiers + { id: id } + end + end + end + end +end diff --git a/lib/gitlab/github_import/single_endpoint_notes_importing.rb b/lib/gitlab/github_import/single_endpoint_notes_importing.rb index 43402ecd165..0a3559adde3 100644 --- a/lib/gitlab/github_import/single_endpoint_notes_importing.rb +++ b/lib/gitlab/github_import/single_endpoint_notes_importing.rb @@ -4,9 +4,14 @@ # - SingleEndpointDiffNotesImporter # - SingleEndpointIssueNotesImporter # - SingleEndpointMergeRequestNotesImporter +# if `github_importer_single_endpoint_notes_import` feature flag is on. # -# `github_importer_single_endpoint_notes_import` -# feature flag is on. +# - SingleEndpointIssueEventsImporter +# if `github_importer_issue_events_import` feature flag is on. +# +# Fetches associated objects page by page to each item of parent collection. +# Currently `associated` is note or event. +# Currently `parent` is MergeRequest or Issue record. # # It fetches 1 PR's associated objects at a time using `issue_comments` or # `pull_request_comments` endpoint, which is slower than `NotesImporter` @@ -18,67 +23,75 @@ module Gitlab module SingleEndpointNotesImporting BATCH_SIZE = 100 - def each_object_to_import - each_notes_page do |page| - page.objects.each do |note| - next if already_imported?(note) + def each_object_to_import(&block) + each_associated_page do |parent_record, associated_page| + associated_page.objects.each do |associated| + each_associated(parent_record, associated, &block) + end + end + end - Gitlab::GithubImport::ObjectCounter.increment(project, object_type, :fetched) + def id_for_already_imported_cache(associated) + associated.id + end - yield(note) + def parent_collection + raise NotImplementedError + end - mark_as_imported(note) - end - end + def parent_imported_cache_key + raise NotImplementedError end - def id_for_already_imported_cache(note) - note.id + def page_counter_id(parent) + raise NotImplementedError end private - def each_notes_page - noteables.each_batch(of: BATCH_SIZE, column: :iid) do |batch| - batch.each do |noteable| - # The page counter needs to be scoped by noteable to avoid skipping - # pages of notes from already imported noteables. - page_counter = PageCounter.new(project, page_counter_id(noteable)) + # Sometimes we need to add some extra info from parent + # to associated record that is not available by default + # in Github API response object. For example: + # lib/gitlab/github_import/importer/single_endpoint_issue_events_importer.rb:26 + def each_associated(_parent_record, associated) + return if already_imported?(associated) + + Gitlab::GithubImport::ObjectCounter.increment(project, object_type, :fetched) + + yield(associated) + + mark_as_imported(associated) + end + + def each_associated_page + parent_collection.each_batch(of: BATCH_SIZE, column: :iid) do |batch| + batch.each do |parent_record| + # The page counter needs to be scoped by parent_record to avoid skipping + # pages of notes from already imported parent_record. + page_counter = PageCounter.new(project, page_counter_id(parent_record)) repo = project.import_source options = collection_options.merge(page: page_counter.current) - client.each_page(collection_method, repo, noteable.iid, options) do |page| + client.each_page(collection_method, repo, parent_record.iid, options) do |page| next unless page_counter.set(page.number) - yield page + yield parent_record, page end - mark_notes_imported(noteable) + mark_parent_imported(parent_record) end end end - def mark_notes_imported(noteable) + def mark_parent_imported(parent) Gitlab::Cache::Import::Caching.set_add( - notes_imported_cache_key, - noteable.iid + parent_imported_cache_key, + parent.iid ) end - def already_imported_noteables - Gitlab::Cache::Import::Caching.values_from_set(notes_imported_cache_key) - end - - def noteables - NotImplementedError - end - - def notes_imported_cache_key - NotImplementedError - end - - def page_counter_id(noteable) - NotImplementedError + def already_imported_parents + Gitlab::Cache::Import::Caching.values_from_set(parent_imported_cache_key) end end end diff --git a/lib/gitlab/github_import/user_finder.rb b/lib/gitlab/github_import/user_finder.rb index 93483ee697a..efaa2ce3002 100644 --- a/lib/gitlab/github_import/user_finder.rb +++ b/lib/gitlab/github_import/user_finder.rb @@ -39,13 +39,9 @@ module Gitlab # # If the object has no author ID we'll use the ID of the GitLab ghost # user. - def author_id_for(object) - id = - if object&.author - user_id_for(object.author) - else - GithubImport.ghost_user_id - end + def author_id_for(object, author_key: :author) + user_info = author_key == :actor ? object&.actor : object&.author + id = user_info ? user_id_for(user_info) : GithubImport.ghost_user_id if id [id, true] |