diff options
Diffstat (limited to 'lib/gitlab/github_import/importer')
14 files changed, 231 insertions, 93 deletions
diff --git a/lib/gitlab/github_import/importer/events/base_importer.rb b/lib/gitlab/github_import/importer/events/base_importer.rb new file mode 100644 index 00000000000..9ab1d916d33 --- /dev/null +++ b/lib/gitlab/github_import/importer/events/base_importer.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Importer + module Events + # Base class for importing issue events during project import from GitHub + class BaseImporter + # project - An instance of `Project`. + # client - An instance of `Gitlab::GithubImport::Client`. + def initialize(project, client) + @project = project + @user_finder = UserFinder.new(project, client) + end + + # issue_event - An instance of `Gitlab::GithubImport::Representation::IssueEvent`. + def execute(issue_event) + raise NotImplementedError + end + + private + + attr_reader :project, :user_finder + + def author_id(issue_event, author_key: :actor) + user_finder.author_id_for(issue_event, author_key: author_key).first + end + + def issuable_db_id(object) + IssuableFinder.new(project, object).database_id + end + end + end + end + end +end diff --git a/lib/gitlab/github_import/importer/events/changed_assignee.rb b/lib/gitlab/github_import/importer/events/changed_assignee.rb new file mode 100644 index 00000000000..c8f6335e4a8 --- /dev/null +++ b/lib/gitlab/github_import/importer/events/changed_assignee.rb @@ -0,0 +1,52 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Importer + module Events + class ChangedAssignee < BaseImporter + def execute(issue_event) + assignee_id = author_id(issue_event, author_key: :assignee) + assigner_id = author_id(issue_event, author_key: :assigner) + + note_body = parse_body(issue_event, assigner_id, assignee_id) + + create_note(issue_event, note_body, assigner_id) + end + + private + + def create_note(issue_event, note_body, assigner_id) + Note.create!( + system: true, + noteable_type: Issue.name, + noteable_id: issuable_db_id(issue_event), + project: project, + author_id: assigner_id, + note: note_body, + system_note_metadata: SystemNoteMetadata.new( + { + action: "assignee", + created_at: issue_event.created_at, + updated_at: issue_event.created_at + } + ), + created_at: issue_event.created_at, + updated_at: issue_event.created_at + ) + end + + def parse_body(issue_event, assigner_id, assignee_id) + Gitlab::I18n.with_default_locale do + if issue_event.event == "unassigned" + "unassigned #{User.find(assigner_id).to_reference}" + else + "assigned to #{User.find(assignee_id).to_reference}" + end + end + end + end + end + end + end +end diff --git a/lib/gitlab/github_import/importer/events/changed_label.rb b/lib/gitlab/github_import/importer/events/changed_label.rb index 6c408158b02..818a9202745 100644 --- a/lib/gitlab/github_import/importer/events/changed_label.rb +++ b/lib/gitlab/github_import/importer/events/changed_label.rb @@ -4,25 +4,17 @@ module Gitlab module GithubImport module Importer module Events - class ChangedLabel - def initialize(project, user_id) - @project = project - @user_id = user_id - end - - # issue_event - An instance of `Gitlab::GithubImport::Representation::IssueEvent`. + class ChangedLabel < BaseImporter def execute(issue_event) create_event(issue_event) end private - attr_reader :project, :user_id - def create_event(issue_event) ResourceLabelEvent.create!( - issue_id: issue_event.issue_db_id, - user_id: user_id, + issue_id: issuable_db_id(issue_event), + user_id: author_id(issue_event), label_id: label_finder.id_for(issue_event.label_title), action: action(issue_event.event), created_at: issue_event.created_at diff --git a/lib/gitlab/github_import/importer/events/changed_milestone.rb b/lib/gitlab/github_import/importer/events/changed_milestone.rb new file mode 100644 index 00000000000..3164c041dc3 --- /dev/null +++ b/lib/gitlab/github_import/importer/events/changed_milestone.rb @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Importer + module Events + class ChangedMilestone < BaseImporter + # GitHub API doesn't provide the historical state of an issue for + # de/milestoned issue events. So we'll assign the default state to + # those events that are imported from GitHub. + DEFAULT_STATE = Issue.available_states[:opened] + + def execute(issue_event) + create_event(issue_event) + end + + private + + def create_event(issue_event) + ResourceMilestoneEvent.create!( + issue_id: issuable_db_id(issue_event), + user_id: author_id(issue_event), + created_at: issue_event.created_at, + milestone_id: project.milestones.find_by_title(issue_event.milestone_title)&.id, + action: action(issue_event.event), + state: DEFAULT_STATE + ) + end + + def action(event_type) + return ResourceMilestoneEvent.actions[:remove] if event_type == 'demilestoned' + + ResourceMilestoneEvent.actions[:add] + end + end + end + end + end +end diff --git a/lib/gitlab/github_import/importer/events/closed.rb b/lib/gitlab/github_import/importer/events/closed.rb index 8b2136c9b24..ca8730d0f27 100644 --- a/lib/gitlab/github_import/importer/events/closed.rb +++ b/lib/gitlab/github_import/importer/events/closed.rb @@ -4,15 +4,7 @@ module Gitlab module GithubImport module Importer module Events - class Closed - attr_reader :project, :user_id - - def initialize(project, user_id) - @project = project - @user_id = user_id - end - - # issue_event - An instance of `Gitlab::GithubImport::Representation::IssueEvent`. + class Closed < BaseImporter def execute(issue_event) create_event(issue_event) create_state_event(issue_event) @@ -23,10 +15,10 @@ module Gitlab def create_event(issue_event) Event.create!( project_id: project.id, - author_id: user_id, + author_id: author_id(issue_event), action: 'closed', target_type: Issue.name, - target_id: issue_event.issue_db_id, + target_id: issuable_db_id(issue_event), created_at: issue_event.created_at, updated_at: issue_event.created_at ) @@ -34,8 +26,8 @@ module Gitlab def create_state_event(issue_event) ResourceStateEvent.create!( - user_id: user_id, - issue_id: issue_event.issue_db_id, + user_id: author_id(issue_event), + issue_id: issuable_db_id(issue_event), source_commit: issue_event.commit_id, state: 'closed', close_after_error_tracking_resolve: false, diff --git a/lib/gitlab/github_import/importer/events/cross_referenced.rb b/lib/gitlab/github_import/importer/events/cross_referenced.rb index 20b902cfe50..89fc1bdeb09 100644 --- a/lib/gitlab/github_import/importer/events/cross_referenced.rb +++ b/lib/gitlab/github_import/importer/events/cross_referenced.rb @@ -4,15 +4,7 @@ module Gitlab module GithubImport module Importer module Events - class CrossReferenced - attr_reader :project, :user_id - - def initialize(project, user_id) - @project = project - @user_id = user_id - end - - # issue_event - An instance of `Gitlab::GithubImport::Representation::IssueEvent`. + class CrossReferenced < BaseImporter def execute(issue_event) mentioned_in_record_class = mentioned_in_type(issue_event) mentioned_in_number = issue_event.source.dig(:issue, :number) @@ -21,14 +13,15 @@ module Gitlab ) return if mentioned_in_record.nil? + user_id = author_id(issue_event) note_body = cross_reference_note_content(mentioned_in_record.gfm_reference(project)) - track_activity(mentioned_in_record_class) - create_note(issue_event, note_body) + track_activity(mentioned_in_record_class, user_id) + create_note(issue_event, note_body, user_id) end private - def track_activity(mentioned_in_class) + def track_activity(mentioned_in_class, user_id) return if mentioned_in_class != Issue Gitlab::UsageDataCounters::HLLRedisCounter.track_event( @@ -37,11 +30,11 @@ module Gitlab ) end - def create_note(issue_event, note_body) + def create_note(issue_event, note_body, user_id) Note.create!( system: true, noteable_type: Issue.name, - noteable_id: issue_event.issue_db_id, + noteable_id: issuable_db_id(issue_event), project: project, author_id: user_id, note: note_body, @@ -73,7 +66,7 @@ module Gitlab iid: number, issuable_type: record_class.name ) - Gitlab::GithubImport::IssuableFinder.new(project, mentioned_in_adapter).database_id + issuable_db_id(mentioned_in_adapter) end def cross_reference_note_content(gfm_reference) diff --git a/lib/gitlab/github_import/importer/events/renamed.rb b/lib/gitlab/github_import/importer/events/renamed.rb index 6a11c492210..96d112b04c6 100644 --- a/lib/gitlab/github_import/importer/events/renamed.rb +++ b/lib/gitlab/github_import/importer/events/renamed.rb @@ -4,27 +4,19 @@ module Gitlab module GithubImport module Importer module Events - class Renamed - def initialize(project, user_id) - @project = project - @user_id = user_id - end - - # issue_event - An instance of `Gitlab::GithubImport::Representation::IssueEvent` + class Renamed < BaseImporter def execute(issue_event) Note.create!(note_params(issue_event)) end private - attr_reader :project, :user_id - def note_params(issue_event) { - noteable_id: issue_event.issue_db_id, + noteable_id: issuable_db_id(issue_event), noteable_type: Issue.name, project_id: project.id, - author_id: user_id, + author_id: author_id(issue_event), note: parse_body(issue_event), system: true, created_at: issue_event.created_at, diff --git a/lib/gitlab/github_import/importer/events/reopened.rb b/lib/gitlab/github_import/importer/events/reopened.rb index c0f3802bc46..b75344bf817 100644 --- a/lib/gitlab/github_import/importer/events/reopened.rb +++ b/lib/gitlab/github_import/importer/events/reopened.rb @@ -4,15 +4,7 @@ module Gitlab module GithubImport module Importer module Events - class Reopened - attr_reader :project, :user_id - - def initialize(project, user_id) - @project = project - @user_id = user_id - end - - # issue_event - An instance of `Gitlab::GithubImport::Representation::IssueEvent`. + class Reopened < BaseImporter def execute(issue_event) create_event(issue_event) create_state_event(issue_event) @@ -23,10 +15,10 @@ module Gitlab def create_event(issue_event) Event.create!( project_id: project.id, - author_id: user_id, + author_id: author_id(issue_event), action: 'reopened', target_type: Issue.name, - target_id: issue_event.issue_db_id, + target_id: issuable_db_id(issue_event), created_at: issue_event.created_at, updated_at: issue_event.created_at ) @@ -34,8 +26,8 @@ module Gitlab def create_state_event(issue_event) ResourceStateEvent.create!( - user_id: user_id, - issue_id: issue_event.issue_db_id, + user_id: author_id(issue_event), + issue_id: issuable_db_id(issue_event), state: 'reopened', created_at: issue_event.created_at ) diff --git a/lib/gitlab/github_import/importer/issue_event_importer.rb b/lib/gitlab/github_import/importer/issue_event_importer.rb index e451af61ec3..ef456e56ee1 100644 --- a/lib/gitlab/github_import/importer/issue_event_importer.rb +++ b/lib/gitlab/github_import/importer/issue_event_importer.rb @@ -4,7 +4,7 @@ module Gitlab module GithubImport module Importer class IssueEventImporter - attr_reader :issue_event, :project, :client, :user_finder + attr_reader :issue_event, :project, :client # issue_event - An instance of `Gitlab::GithubImport::Representation::IssueEvent`. # project - An instance of `Project`. @@ -13,26 +13,16 @@ module Gitlab @issue_event = issue_event @project = project @client = client - @user_finder = UserFinder.new(project, client) end + # TODO: Add MergeRequest events support + # https://gitlab.com/groups/gitlab-org/-/epics/7673 def execute - case issue_event.event - when 'closed' - Gitlab::GithubImport::Importer::Events::Closed.new(project, author_id) - .execute(issue_event) - when 'reopened' - Gitlab::GithubImport::Importer::Events::Reopened.new(project, author_id) - .execute(issue_event) - when 'labeled', 'unlabeled' - Gitlab::GithubImport::Importer::Events::ChangedLabel.new(project, author_id) - .execute(issue_event) - when 'renamed' - Gitlab::GithubImport::Importer::Events::Renamed.new(project, author_id) - .execute(issue_event) - when 'cross-referenced' - Gitlab::GithubImport::Importer::Events::CrossReferenced.new(project, author_id) - .execute(issue_event) + return if issue_event.issuable_type == 'MergeRequest' + + importer = event_importer_class(issue_event) + if importer + importer.new(project, client).execute(issue_event) else Gitlab::GithubImport::Logger.debug( message: 'UNSUPPORTED_EVENT_TYPE', @@ -43,9 +33,23 @@ module Gitlab private - def author_id - id, _status = user_finder.author_id_for(issue_event, author_key: :actor) - id + def event_importer_class(issue_event) + case issue_event.event + when 'closed' + Gitlab::GithubImport::Importer::Events::Closed + when 'reopened' + Gitlab::GithubImport::Importer::Events::Reopened + when 'labeled', 'unlabeled' + Gitlab::GithubImport::Importer::Events::ChangedLabel + when 'renamed' + Gitlab::GithubImport::Importer::Events::Renamed + when 'milestoned', 'demilestoned' + Gitlab::GithubImport::Importer::Events::ChangedMilestone + when 'cross-referenced' + Gitlab::GithubImport::Importer::Events::CrossReferenced + when 'assigned', 'unassigned' + Gitlab::GithubImport::Importer::Events::ChangedAssignee + end end end end diff --git a/lib/gitlab/github_import/importer/issue_events_importer.rb b/lib/gitlab/github_import/importer/issue_events_importer.rb new file mode 100644 index 00000000000..71dd99f91f9 --- /dev/null +++ b/lib/gitlab/github_import/importer/issue_events_importer.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Importer + class IssueEventsImporter + include ParallelScheduling + + def importer_class + IssueEventImporter + end + + def representation_class + Representation::IssueEvent + end + + def sidekiq_worker_class + ImportIssueEventWorker + end + + def object_type + :issue_event + end + + def collection_method + :repository_issue_events + end + + def id_for_already_imported_cache(event) + event.id + end + end + end + end +end diff --git a/lib/gitlab/github_import/importer/issue_importer.rb b/lib/gitlab/github_import/importer/issue_importer.rb index e7d41856b04..d964bae3dd2 100644 --- a/lib/gitlab/github_import/importer/issue_importer.rb +++ b/lib/gitlab/github_import/importer/issue_importer.rb @@ -56,7 +56,8 @@ module Gitlab milestone_id: milestone_finder.id_for(issue), state_id: ::Issue.available_states[issue.state], created_at: issue.created_at, - updated_at: issue.updated_at + updated_at: issue.updated_at, + work_item_type_id: issue.work_item_type_id } insert_and_return_id(attributes, project.issues) diff --git a/lib/gitlab/github_import/importer/issues_importer.rb b/lib/gitlab/github_import/importer/issues_importer.rb index 6cc1a61b332..21d9ce8cd2d 100644 --- a/lib/gitlab/github_import/importer/issues_importer.rb +++ b/lib/gitlab/github_import/importer/issues_importer.rb @@ -6,6 +6,12 @@ module Gitlab class IssuesImporter include ParallelScheduling + def initialize(project, client, parallel: true) + super + + @work_item_type_id = ::WorkItems::Type.default_issue_type.id + end + def importer_class IssueAndLabelLinksImporter end @@ -33,6 +39,12 @@ module Gitlab def collection_options { state: 'all', sort: 'created', direction: 'asc' } end + + private + + def additional_object_data + { work_item_type_id: @work_item_type_id } + end end end end diff --git a/lib/gitlab/github_import/importer/releases_importer.rb b/lib/gitlab/github_import/importer/releases_importer.rb index 7241e1ef703..51d364772d2 100644 --- a/lib/gitlab/github_import/importer/releases_importer.rb +++ b/lib/gitlab/github_import/importer/releases_importer.rb @@ -27,9 +27,10 @@ module Gitlab def build(release) existing_tags.add(release.tag_name) - build_hash = { + { name: release.name, tag: release.tag_name, + author_id: fetch_author_id(release), description: description_for(release), created_at: release.created_at, updated_at: release.created_at, @@ -37,12 +38,6 @@ module Gitlab released_at: release.published_at || Time.current, project_id: project.id } - - if Feature.enabled?(:import_release_authors_from_github, project) - build_hash[:author_id] = fetch_author_id(release) - end - - build_hash end def each_release diff --git a/lib/gitlab/github_import/importer/single_endpoint_issue_events_importer.rb b/lib/gitlab/github_import/importer/single_endpoint_issue_events_importer.rb index 45bbc25e637..8e4015acbbc 100644 --- a/lib/gitlab/github_import/importer/single_endpoint_issue_events_importer.rb +++ b/lib/gitlab/github_import/importer/single_endpoint_issue_events_importer.rb @@ -18,13 +18,16 @@ module Gitlab { project: project.id, collection: collection_method } end + # In single endpoint there is no issue info to which associated related + # To make it possible to identify issue in separated worker we need to patch + # Sawyer instances here with issue number def each_associated(parent_record, associated) compose_associated_id!(parent_record, associated) return if already_imported?(associated) Gitlab::GithubImport::ObjectCounter.increment(project, object_type, :fetched) - associated.issue_db_id = parent_record.id + associated.issue = { 'number' => parent_record.iid } yield(associated) mark_as_imported(associated) |