Welcome to mirror list, hosted at ThFree Co, Russian Federation.

gitlab.com/gitlab-org/gitlab-foss.git - Unnamed repository; edit this file 'description' to name the repository.
summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorGitLab Bot <gitlab-bot@gitlab.com>2022-07-20 18:40:28 +0300
committerGitLab Bot <gitlab-bot@gitlab.com>2022-07-20 18:40:28 +0300
commitb595cb0c1dec83de5bdee18284abe86614bed33b (patch)
tree8c3d4540f193c5ff98019352f554e921b3a41a72 /lib/gitlab/github_import
parent2f9104a328fc8a4bddeaa4627b595166d24671d0 (diff)
Add latest changes from gitlab-org/gitlab@15-2-stable-eev15.2.0-rc42
Diffstat (limited to 'lib/gitlab/github_import')
-rw-r--r--lib/gitlab/github_import/importer/events/changed_label.rb43
-rw-r--r--lib/gitlab/github_import/importer/events/closed.rb50
-rw-r--r--lib/gitlab/github_import/importer/events/cross_referenced.rb86
-rw-r--r--lib/gitlab/github_import/importer/events/renamed.rb56
-rw-r--r--lib/gitlab/github_import/importer/events/reopened.rb47
-rw-r--r--lib/gitlab/github_import/importer/issue_event_importer.rb53
-rw-r--r--lib/gitlab/github_import/importer/issue_importer.rb1
-rw-r--r--lib/gitlab/github_import/importer/note_importer.rb11
-rw-r--r--lib/gitlab/github_import/importer/single_endpoint_diff_notes_importer.rb6
-rw-r--r--lib/gitlab/github_import/importer/single_endpoint_issue_events_importer.rb82
-rw-r--r--lib/gitlab/github_import/importer/single_endpoint_issue_notes_importer.rb6
-rw-r--r--lib/gitlab/github_import/importer/single_endpoint_merge_request_notes_importer.rb6
-rw-r--r--lib/gitlab/github_import/markdown_text.rb30
-rw-r--r--lib/gitlab/github_import/representation/issue_event.rb54
-rw-r--r--lib/gitlab/github_import/single_endpoint_notes_importing.rb91
-rw-r--r--lib/gitlab/github_import/user_finder.rb10
16 files changed, 572 insertions, 60 deletions
diff --git a/lib/gitlab/github_import/importer/events/changed_label.rb b/lib/gitlab/github_import/importer/events/changed_label.rb
new file mode 100644
index 00000000000..6c408158b02
--- /dev/null
+++ b/lib/gitlab/github_import/importer/events/changed_label.rb
@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module GithubImport
+ module Importer
+ module Events
+ class ChangedLabel
+ def initialize(project, user_id)
+ @project = project
+ @user_id = user_id
+ end
+
+ # issue_event - An instance of `Gitlab::GithubImport::Representation::IssueEvent`.
+ def execute(issue_event)
+ create_event(issue_event)
+ end
+
+ private
+
+ attr_reader :project, :user_id
+
+ def create_event(issue_event)
+ ResourceLabelEvent.create!(
+ issue_id: issue_event.issue_db_id,
+ user_id: user_id,
+ label_id: label_finder.id_for(issue_event.label_title),
+ action: action(issue_event.event),
+ created_at: issue_event.created_at
+ )
+ end
+
+ def label_finder
+ Gitlab::GithubImport::LabelFinder.new(project)
+ end
+
+ def action(event_type)
+ event_type == 'unlabeled' ? 'remove' : 'add'
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/github_import/importer/events/closed.rb b/lib/gitlab/github_import/importer/events/closed.rb
new file mode 100644
index 00000000000..8b2136c9b24
--- /dev/null
+++ b/lib/gitlab/github_import/importer/events/closed.rb
@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module GithubImport
+ module Importer
+ module Events
+ class Closed
+ attr_reader :project, :user_id
+
+ def initialize(project, user_id)
+ @project = project
+ @user_id = user_id
+ end
+
+ # issue_event - An instance of `Gitlab::GithubImport::Representation::IssueEvent`.
+ def execute(issue_event)
+ create_event(issue_event)
+ create_state_event(issue_event)
+ end
+
+ private
+
+ def create_event(issue_event)
+ Event.create!(
+ project_id: project.id,
+ author_id: user_id,
+ action: 'closed',
+ target_type: Issue.name,
+ target_id: issue_event.issue_db_id,
+ created_at: issue_event.created_at,
+ updated_at: issue_event.created_at
+ )
+ end
+
+ def create_state_event(issue_event)
+ ResourceStateEvent.create!(
+ user_id: user_id,
+ issue_id: issue_event.issue_db_id,
+ source_commit: issue_event.commit_id,
+ state: 'closed',
+ close_after_error_tracking_resolve: false,
+ close_auto_resolve_prometheus_alert: false,
+ created_at: issue_event.created_at
+ )
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/github_import/importer/events/cross_referenced.rb b/lib/gitlab/github_import/importer/events/cross_referenced.rb
new file mode 100644
index 00000000000..20b902cfe50
--- /dev/null
+++ b/lib/gitlab/github_import/importer/events/cross_referenced.rb
@@ -0,0 +1,86 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module GithubImport
+ module Importer
+ module Events
+ class CrossReferenced
+ attr_reader :project, :user_id
+
+ def initialize(project, user_id)
+ @project = project
+ @user_id = user_id
+ end
+
+ # issue_event - An instance of `Gitlab::GithubImport::Representation::IssueEvent`.
+ def execute(issue_event)
+ mentioned_in_record_class = mentioned_in_type(issue_event)
+ mentioned_in_number = issue_event.source.dig(:issue, :number)
+ mentioned_in_record = init_mentioned_in(
+ mentioned_in_record_class, mentioned_in_number
+ )
+ return if mentioned_in_record.nil?
+
+ note_body = cross_reference_note_content(mentioned_in_record.gfm_reference(project))
+ track_activity(mentioned_in_record_class)
+ create_note(issue_event, note_body)
+ end
+
+ private
+
+ def track_activity(mentioned_in_class)
+ return if mentioned_in_class != Issue
+
+ Gitlab::UsageDataCounters::HLLRedisCounter.track_event(
+ Gitlab::UsageDataCounters::IssueActivityUniqueCounter::ISSUE_CROSS_REFERENCED,
+ values: user_id
+ )
+ end
+
+ def create_note(issue_event, note_body)
+ Note.create!(
+ system: true,
+ noteable_type: Issue.name,
+ noteable_id: issue_event.issue_db_id,
+ project: project,
+ author_id: user_id,
+ note: note_body,
+ system_note_metadata: SystemNoteMetadata.new(action: 'cross_reference'),
+ created_at: issue_event.created_at
+ )
+ end
+
+ def mentioned_in_type(issue_event)
+ is_pull_request = issue_event.source.dig(:issue, :pull_request).present?
+ is_pull_request ? MergeRequest : Issue
+ end
+
+ # record_class - Issue/MergeRequest
+ def init_mentioned_in(record_class, iid)
+ db_id = fetch_mentioned_in_db_id(record_class, iid)
+ return if db_id.nil?
+
+ record = record_class.new(id: db_id, iid: iid)
+ record.project = project
+ record.readonly!
+ record
+ end
+
+ # record_class - Issue/MergeRequest
+ def fetch_mentioned_in_db_id(record_class, number)
+ sawyer_mentioned_in_adapter = Struct.new(:iid, :issuable_type, keyword_init: true)
+ mentioned_in_adapter = sawyer_mentioned_in_adapter.new(
+ iid: number, issuable_type: record_class.name
+ )
+
+ Gitlab::GithubImport::IssuableFinder.new(project, mentioned_in_adapter).database_id
+ end
+
+ def cross_reference_note_content(gfm_reference)
+ "#{::SystemNotes::IssuablesService.cross_reference_note_prefix}#{gfm_reference}"
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/github_import/importer/events/renamed.rb b/lib/gitlab/github_import/importer/events/renamed.rb
new file mode 100644
index 00000000000..6a11c492210
--- /dev/null
+++ b/lib/gitlab/github_import/importer/events/renamed.rb
@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module GithubImport
+ module Importer
+ module Events
+ class Renamed
+ def initialize(project, user_id)
+ @project = project
+ @user_id = user_id
+ end
+
+ # issue_event - An instance of `Gitlab::GithubImport::Representation::IssueEvent`
+ def execute(issue_event)
+ Note.create!(note_params(issue_event))
+ end
+
+ private
+
+ attr_reader :project, :user_id
+
+ def note_params(issue_event)
+ {
+ noteable_id: issue_event.issue_db_id,
+ noteable_type: Issue.name,
+ project_id: project.id,
+ author_id: user_id,
+ note: parse_body(issue_event),
+ system: true,
+ created_at: issue_event.created_at,
+ updated_at: issue_event.created_at,
+ system_note_metadata: SystemNoteMetadata.new(
+ {
+ action: "title",
+ created_at: issue_event.created_at,
+ updated_at: issue_event.created_at
+ }
+ )
+ }
+ end
+
+ def parse_body(issue_event)
+ old_diffs, new_diffs = Gitlab::Diff::InlineDiff.new(
+ issue_event.old_title, issue_event.new_title
+ ).inline_diffs
+
+ marked_old_title = Gitlab::Diff::InlineDiffMarkdownMarker.new(issue_event.old_title).mark(old_diffs)
+ marked_new_title = Gitlab::Diff::InlineDiffMarkdownMarker.new(issue_event.new_title).mark(new_diffs)
+
+ "changed title from **#{marked_old_title}** to **#{marked_new_title}**"
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/github_import/importer/events/reopened.rb b/lib/gitlab/github_import/importer/events/reopened.rb
new file mode 100644
index 00000000000..c0f3802bc46
--- /dev/null
+++ b/lib/gitlab/github_import/importer/events/reopened.rb
@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module GithubImport
+ module Importer
+ module Events
+ class Reopened
+ attr_reader :project, :user_id
+
+ def initialize(project, user_id)
+ @project = project
+ @user_id = user_id
+ end
+
+ # issue_event - An instance of `Gitlab::GithubImport::Representation::IssueEvent`.
+ def execute(issue_event)
+ create_event(issue_event)
+ create_state_event(issue_event)
+ end
+
+ private
+
+ def create_event(issue_event)
+ Event.create!(
+ project_id: project.id,
+ author_id: user_id,
+ action: 'reopened',
+ target_type: Issue.name,
+ target_id: issue_event.issue_db_id,
+ created_at: issue_event.created_at,
+ updated_at: issue_event.created_at
+ )
+ end
+
+ def create_state_event(issue_event)
+ ResourceStateEvent.create!(
+ user_id: user_id,
+ issue_id: issue_event.issue_db_id,
+ state: 'reopened',
+ created_at: issue_event.created_at
+ )
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/github_import/importer/issue_event_importer.rb b/lib/gitlab/github_import/importer/issue_event_importer.rb
new file mode 100644
index 00000000000..e451af61ec3
--- /dev/null
+++ b/lib/gitlab/github_import/importer/issue_event_importer.rb
@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module GithubImport
+ module Importer
+ class IssueEventImporter
+ attr_reader :issue_event, :project, :client, :user_finder
+
+ # issue_event - An instance of `Gitlab::GithubImport::Representation::IssueEvent`.
+ # project - An instance of `Project`.
+ # client - An instance of `Gitlab::GithubImport::Client`.
+ def initialize(issue_event, project, client)
+ @issue_event = issue_event
+ @project = project
+ @client = client
+ @user_finder = UserFinder.new(project, client)
+ end
+
+ def execute
+ case issue_event.event
+ when 'closed'
+ Gitlab::GithubImport::Importer::Events::Closed.new(project, author_id)
+ .execute(issue_event)
+ when 'reopened'
+ Gitlab::GithubImport::Importer::Events::Reopened.new(project, author_id)
+ .execute(issue_event)
+ when 'labeled', 'unlabeled'
+ Gitlab::GithubImport::Importer::Events::ChangedLabel.new(project, author_id)
+ .execute(issue_event)
+ when 'renamed'
+ Gitlab::GithubImport::Importer::Events::Renamed.new(project, author_id)
+ .execute(issue_event)
+ when 'cross-referenced'
+ Gitlab::GithubImport::Importer::Events::CrossReferenced.new(project, author_id)
+ .execute(issue_event)
+ else
+ Gitlab::GithubImport::Logger.debug(
+ message: 'UNSUPPORTED_EVENT_TYPE',
+ event_type: issue_event.event, event_github_id: issue_event.id
+ )
+ end
+ end
+
+ private
+
+ def author_id
+ id, _status = user_finder.author_id_for(issue_event, author_key: :actor)
+ id
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/github_import/importer/issue_importer.rb b/lib/gitlab/github_import/importer/issue_importer.rb
index 35fd4bd88a0..e7d41856b04 100644
--- a/lib/gitlab/github_import/importer/issue_importer.rb
+++ b/lib/gitlab/github_import/importer/issue_importer.rb
@@ -51,6 +51,7 @@ module Gitlab
title: issue.truncated_title,
author_id: author_id,
project_id: project.id,
+ namespace_id: project.project_namespace_id,
description: description,
milestone_id: milestone_finder.id_for(issue),
state_id: ::Issue.available_states[issue.state],
diff --git a/lib/gitlab/github_import/importer/note_importer.rb b/lib/gitlab/github_import/importer/note_importer.rb
index 673f56b5753..1410006af26 100644
--- a/lib/gitlab/github_import/importer/note_importer.rb
+++ b/lib/gitlab/github_import/importer/note_importer.rb
@@ -21,14 +21,12 @@ module Gitlab
author_id, author_found = user_finder.author_id_for(note)
- note_body = MarkdownText.format(note.note, note.author, author_found)
-
attributes = {
noteable_type: note.noteable_type,
noteable_id: noteable_id,
project_id: project.id,
author_id: author_id,
- note: note_body,
+ note: note_body(author_found),
discussion_id: note.discussion_id,
system: false,
created_at: note.created_at,
@@ -48,6 +46,13 @@ module Gitlab
def find_noteable_id
GithubImport::IssuableFinder.new(project, note).database_id
end
+
+ private
+
+ def note_body(author_found)
+ text = MarkdownText.convert_ref_links(note.note, project)
+ MarkdownText.format(text, note.author, author_found)
+ end
end
end
end
diff --git a/lib/gitlab/github_import/importer/single_endpoint_diff_notes_importer.rb b/lib/gitlab/github_import/importer/single_endpoint_diff_notes_importer.rb
index a2c3d1bd057..e1d9ae44065 100644
--- a/lib/gitlab/github_import/importer/single_endpoint_diff_notes_importer.rb
+++ b/lib/gitlab/github_import/importer/single_endpoint_diff_notes_importer.rb
@@ -37,15 +37,15 @@ module Gitlab
private
- def noteables
- project.merge_requests.where.not(iid: already_imported_noteables) # rubocop: disable CodeReuse/ActiveRecord
+ def parent_collection
+ project.merge_requests.where.not(iid: already_imported_parents) # rubocop: disable CodeReuse/ActiveRecord
end
def page_counter_id(merge_request)
"merge_request/#{merge_request.id}/#{collection_method}"
end
- def notes_imported_cache_key
+ def parent_imported_cache_key
"github-importer/merge_request/diff_notes/already-imported/#{project.id}"
end
end
diff --git a/lib/gitlab/github_import/importer/single_endpoint_issue_events_importer.rb b/lib/gitlab/github_import/importer/single_endpoint_issue_events_importer.rb
new file mode 100644
index 00000000000..45bbc25e637
--- /dev/null
+++ b/lib/gitlab/github_import/importer/single_endpoint_issue_events_importer.rb
@@ -0,0 +1,82 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module GithubImport
+ module Importer
+ class SingleEndpointIssueEventsImporter
+ include ParallelScheduling
+ include SingleEndpointNotesImporting
+
+ PROCESSED_PAGE_CACHE_KEY = 'issues/%{issue_iid}/%{collection}'
+ BATCH_SIZE = 100
+
+ def initialize(project, client, parallel: true)
+ @project = project
+ @client = client
+ @parallel = parallel
+ @already_imported_cache_key = ALREADY_IMPORTED_CACHE_KEY %
+ { project: project.id, collection: collection_method }
+ end
+
+ def each_associated(parent_record, associated)
+ compose_associated_id!(parent_record, associated)
+ return if already_imported?(associated)
+
+ Gitlab::GithubImport::ObjectCounter.increment(project, object_type, :fetched)
+
+ associated.issue_db_id = parent_record.id
+ yield(associated)
+
+ mark_as_imported(associated)
+ end
+
+ def importer_class
+ IssueEventImporter
+ end
+
+ def representation_class
+ Representation::IssueEvent
+ end
+
+ def sidekiq_worker_class
+ ImportIssueEventWorker
+ end
+
+ def object_type
+ :issue_event
+ end
+
+ def collection_method
+ :issue_timeline
+ end
+
+ def parent_collection
+ project.issues.where.not(iid: already_imported_parents).select(:id, :iid) # rubocop: disable CodeReuse/ActiveRecord
+ end
+
+ def parent_imported_cache_key
+ "github-importer/issues/#{collection_method}/already-imported/#{project.id}"
+ end
+
+ def page_counter_id(issue)
+ PROCESSED_PAGE_CACHE_KEY % { issue_iid: issue.iid, collection: collection_method }
+ end
+
+ def id_for_already_imported_cache(event)
+ event.id
+ end
+
+ def collection_options
+ { state: 'all', sort: 'created', direction: 'asc' }
+ end
+
+ # Cross-referenced events on Github doesn't have id.
+ def compose_associated_id!(issue, event)
+ return if event.event != 'cross-referenced'
+
+ event.id = "cross-reference##{issue.id}-in-#{event.source.issue.id}"
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/github_import/importer/single_endpoint_issue_notes_importer.rb b/lib/gitlab/github_import/importer/single_endpoint_issue_notes_importer.rb
index 49569ed52d8..fe64df45700 100644
--- a/lib/gitlab/github_import/importer/single_endpoint_issue_notes_importer.rb
+++ b/lib/gitlab/github_import/importer/single_endpoint_issue_notes_importer.rb
@@ -37,15 +37,15 @@ module Gitlab
private
- def noteables
- project.issues.where.not(iid: already_imported_noteables) # rubocop: disable CodeReuse/ActiveRecord
+ def parent_collection
+ project.issues.where.not(iid: already_imported_parents) # rubocop: disable CodeReuse/ActiveRecord
end
def page_counter_id(issue)
"issue/#{issue.id}/#{collection_method}"
end
- def notes_imported_cache_key
+ def parent_imported_cache_key
"github-importer/issue/notes/already-imported/#{project.id}"
end
end
diff --git a/lib/gitlab/github_import/importer/single_endpoint_merge_request_notes_importer.rb b/lib/gitlab/github_import/importer/single_endpoint_merge_request_notes_importer.rb
index d837639c14d..3b1991d2b88 100644
--- a/lib/gitlab/github_import/importer/single_endpoint_merge_request_notes_importer.rb
+++ b/lib/gitlab/github_import/importer/single_endpoint_merge_request_notes_importer.rb
@@ -37,15 +37,15 @@ module Gitlab
private
- def noteables
- project.merge_requests.where.not(iid: already_imported_noteables) # rubocop: disable CodeReuse/ActiveRecord
+ def parent_collection
+ project.merge_requests.where.not(iid: already_imported_parents) # rubocop: disable CodeReuse/ActiveRecord
end
def page_counter_id(merge_request)
"merge_request/#{merge_request.id}/#{collection_method}"
end
- def notes_imported_cache_key
+ def parent_imported_cache_key
"github-importer/merge_request/notes/already-imported/#{project.id}"
end
end
diff --git a/lib/gitlab/github_import/markdown_text.rb b/lib/gitlab/github_import/markdown_text.rb
index 0b1c221bbec..692016bd005 100644
--- a/lib/gitlab/github_import/markdown_text.rb
+++ b/lib/gitlab/github_import/markdown_text.rb
@@ -5,8 +5,34 @@ module Gitlab
class MarkdownText
include Gitlab::EncodingHelper
- def self.format(*args)
- new(*args).to_s
+ ISSUE_REF_MATCHER = '%{github_url}/%{import_source}/issues'
+ PULL_REF_MATCHER = '%{github_url}/%{import_source}/pull'
+
+ class << self
+ def format(*args)
+ new(*args).to_s
+ end
+
+ # Links like `https://domain.github.com/<namespace>/<project>/pull/<iid>` needs to be converted
+ def convert_ref_links(text, project)
+ matcher_options = { github_url: github_url, import_source: project.import_source }
+ issue_ref_matcher = ISSUE_REF_MATCHER % matcher_options
+ pull_ref_matcher = PULL_REF_MATCHER % matcher_options
+
+ url_helpers = Rails.application.routes.url_helpers
+ text.gsub(issue_ref_matcher, url_helpers.project_issues_url(project))
+ .gsub(pull_ref_matcher, url_helpers.project_merge_requests_url(project))
+ end
+
+ private
+
+ # Returns github domain without slash in the end
+ def github_url
+ oauth_config = Gitlab::Auth::OAuth::Provider.config_for('github') || {}
+ url = oauth_config['url'].presence || 'https://github.com'
+ url = url.chop if url.end_with?('/')
+ url
+ end
end
# text - The Markdown text as a String.
diff --git a/lib/gitlab/github_import/representation/issue_event.rb b/lib/gitlab/github_import/representation/issue_event.rb
new file mode 100644
index 00000000000..9016338db3b
--- /dev/null
+++ b/lib/gitlab/github_import/representation/issue_event.rb
@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module GithubImport
+ module Representation
+ class IssueEvent
+ include ToHash
+ include ExposeAttribute
+
+ attr_reader :attributes
+
+ expose_attribute :id, :actor, :event, :commit_id, :label_title, :old_title, :new_title,
+ :source, :created_at
+ expose_attribute :issue_db_id # set in SingleEndpointIssueEventsImporter#each_associated
+
+ # Builds a event from a GitHub API response.
+ #
+ # event - An instance of `Sawyer::Resource` containing the event details.
+ def self.from_api_response(event)
+ new(
+ id: event.id,
+ actor: event.actor && Representation::User.from_api_response(event.actor),
+ event: event.event,
+ commit_id: event.commit_id,
+ label_title: event.label && event.label[:name],
+ old_title: event.rename && event.rename[:from],
+ new_title: event.rename && event.rename[:to],
+ source: event.source,
+ issue_db_id: event.issue_db_id,
+ created_at: event.created_at
+ )
+ end
+
+ # Builds a event using a Hash that was built from a JSON payload.
+ def self.from_json_hash(raw_hash)
+ hash = Representation.symbolize_hash(raw_hash)
+ hash[:actor] &&= Representation::User.from_json_hash(hash[:actor])
+
+ new(hash)
+ end
+
+ # attributes - A Hash containing the event details. The keys of this
+ # Hash (and any nested hashes) must be symbols.
+ def initialize(attributes)
+ @attributes = attributes
+ end
+
+ def github_identifiers
+ { id: id }
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/github_import/single_endpoint_notes_importing.rb b/lib/gitlab/github_import/single_endpoint_notes_importing.rb
index 43402ecd165..0a3559adde3 100644
--- a/lib/gitlab/github_import/single_endpoint_notes_importing.rb
+++ b/lib/gitlab/github_import/single_endpoint_notes_importing.rb
@@ -4,9 +4,14 @@
# - SingleEndpointDiffNotesImporter
# - SingleEndpointIssueNotesImporter
# - SingleEndpointMergeRequestNotesImporter
+# if `github_importer_single_endpoint_notes_import` feature flag is on.
#
-# `github_importer_single_endpoint_notes_import`
-# feature flag is on.
+# - SingleEndpointIssueEventsImporter
+# if `github_importer_issue_events_import` feature flag is on.
+#
+# Fetches associated objects page by page to each item of parent collection.
+# Currently `associated` is note or event.
+# Currently `parent` is MergeRequest or Issue record.
#
# It fetches 1 PR's associated objects at a time using `issue_comments` or
# `pull_request_comments` endpoint, which is slower than `NotesImporter`
@@ -18,67 +23,75 @@ module Gitlab
module SingleEndpointNotesImporting
BATCH_SIZE = 100
- def each_object_to_import
- each_notes_page do |page|
- page.objects.each do |note|
- next if already_imported?(note)
+ def each_object_to_import(&block)
+ each_associated_page do |parent_record, associated_page|
+ associated_page.objects.each do |associated|
+ each_associated(parent_record, associated, &block)
+ end
+ end
+ end
- Gitlab::GithubImport::ObjectCounter.increment(project, object_type, :fetched)
+ def id_for_already_imported_cache(associated)
+ associated.id
+ end
- yield(note)
+ def parent_collection
+ raise NotImplementedError
+ end
- mark_as_imported(note)
- end
- end
+ def parent_imported_cache_key
+ raise NotImplementedError
end
- def id_for_already_imported_cache(note)
- note.id
+ def page_counter_id(parent)
+ raise NotImplementedError
end
private
- def each_notes_page
- noteables.each_batch(of: BATCH_SIZE, column: :iid) do |batch|
- batch.each do |noteable|
- # The page counter needs to be scoped by noteable to avoid skipping
- # pages of notes from already imported noteables.
- page_counter = PageCounter.new(project, page_counter_id(noteable))
+ # Sometimes we need to add some extra info from parent
+ # to associated record that is not available by default
+ # in Github API response object. For example:
+ # lib/gitlab/github_import/importer/single_endpoint_issue_events_importer.rb:26
+ def each_associated(_parent_record, associated)
+ return if already_imported?(associated)
+
+ Gitlab::GithubImport::ObjectCounter.increment(project, object_type, :fetched)
+
+ yield(associated)
+
+ mark_as_imported(associated)
+ end
+
+ def each_associated_page
+ parent_collection.each_batch(of: BATCH_SIZE, column: :iid) do |batch|
+ batch.each do |parent_record|
+ # The page counter needs to be scoped by parent_record to avoid skipping
+ # pages of notes from already imported parent_record.
+ page_counter = PageCounter.new(project, page_counter_id(parent_record))
repo = project.import_source
options = collection_options.merge(page: page_counter.current)
- client.each_page(collection_method, repo, noteable.iid, options) do |page|
+ client.each_page(collection_method, repo, parent_record.iid, options) do |page|
next unless page_counter.set(page.number)
- yield page
+ yield parent_record, page
end
- mark_notes_imported(noteable)
+ mark_parent_imported(parent_record)
end
end
end
- def mark_notes_imported(noteable)
+ def mark_parent_imported(parent)
Gitlab::Cache::Import::Caching.set_add(
- notes_imported_cache_key,
- noteable.iid
+ parent_imported_cache_key,
+ parent.iid
)
end
- def already_imported_noteables
- Gitlab::Cache::Import::Caching.values_from_set(notes_imported_cache_key)
- end
-
- def noteables
- NotImplementedError
- end
-
- def notes_imported_cache_key
- NotImplementedError
- end
-
- def page_counter_id(noteable)
- NotImplementedError
+ def already_imported_parents
+ Gitlab::Cache::Import::Caching.values_from_set(parent_imported_cache_key)
end
end
end
diff --git a/lib/gitlab/github_import/user_finder.rb b/lib/gitlab/github_import/user_finder.rb
index 93483ee697a..efaa2ce3002 100644
--- a/lib/gitlab/github_import/user_finder.rb
+++ b/lib/gitlab/github_import/user_finder.rb
@@ -39,13 +39,9 @@ module Gitlab
#
# If the object has no author ID we'll use the ID of the GitLab ghost
# user.
- def author_id_for(object)
- id =
- if object&.author
- user_id_for(object.author)
- else
- GithubImport.ghost_user_id
- end
+ def author_id_for(object, author_key: :author)
+ user_info = author_key == :actor ? object&.actor : object&.author
+ id = user_info ? user_id_for(user_info) : GithubImport.ghost_user_id
if id
[id, true]