diff options
Diffstat (limited to 'lib/gitlab/github_import/importer')
16 files changed, 338 insertions, 38 deletions
diff --git a/lib/gitlab/github_import/importer/events/base_importer.rb b/lib/gitlab/github_import/importer/events/base_importer.rb index 9ab1d916d33..8218acf2bfb 100644 --- a/lib/gitlab/github_import/importer/events/base_importer.rb +++ b/lib/gitlab/github_import/importer/events/base_importer.rb @@ -29,6 +29,19 @@ module Gitlab def issuable_db_id(object) IssuableFinder.new(project, object).database_id end + + def issuable_type(issue_event) + merge_request_event?(issue_event) ? MergeRequest.name : Issue.name + end + + def merge_request_event?(issue_event) + issue_event.issuable_type == MergeRequest.name + end + + def resource_event_belongs_to(issue_event) + belongs_to_key = merge_request_event?(issue_event) ? :merge_request_id : :issue_id + { belongs_to_key => issuable_db_id(issue_event) } + end end end end diff --git a/lib/gitlab/github_import/importer/events/changed_assignee.rb b/lib/gitlab/github_import/importer/events/changed_assignee.rb index c8f6335e4a8..b75d41f40de 100644 --- a/lib/gitlab/github_import/importer/events/changed_assignee.rb +++ b/lib/gitlab/github_import/importer/events/changed_assignee.rb @@ -7,22 +7,22 @@ module Gitlab class ChangedAssignee < BaseImporter def execute(issue_event) assignee_id = author_id(issue_event, author_key: :assignee) - assigner_id = author_id(issue_event, author_key: :assigner) + author_id = author_id(issue_event, author_key: :actor) - note_body = parse_body(issue_event, assigner_id, assignee_id) + note_body = parse_body(issue_event, assignee_id) - create_note(issue_event, note_body, assigner_id) + create_note(issue_event, note_body, author_id) end private - def create_note(issue_event, note_body, assigner_id) + def create_note(issue_event, note_body, author_id) Note.create!( system: true, - noteable_type: Issue.name, + noteable_type: issuable_type(issue_event), noteable_id: issuable_db_id(issue_event), project: project, - author_id: assigner_id, + author_id: author_id, note: note_body, system_note_metadata: SystemNoteMetadata.new( { @@ -36,12 +36,14 @@ module Gitlab ) end - def parse_body(issue_event, assigner_id, assignee_id) + def parse_body(issue_event, assignee_id) + assignee = User.find(assignee_id).to_reference + Gitlab::I18n.with_default_locale do if issue_event.event == "unassigned" - "unassigned #{User.find(assigner_id).to_reference}" + "unassigned #{assignee}" else - "assigned to #{User.find(assignee_id).to_reference}" + "assigned to #{assignee}" end end end diff --git a/lib/gitlab/github_import/importer/events/changed_label.rb b/lib/gitlab/github_import/importer/events/changed_label.rb index 818a9202745..83130d18db9 100644 --- a/lib/gitlab/github_import/importer/events/changed_label.rb +++ b/lib/gitlab/github_import/importer/events/changed_label.rb @@ -12,13 +12,14 @@ module Gitlab private def create_event(issue_event) - ResourceLabelEvent.create!( - issue_id: issuable_db_id(issue_event), + attrs = { user_id: author_id(issue_event), label_id: label_finder.id_for(issue_event.label_title), action: action(issue_event.event), created_at: issue_event.created_at - ) + }.merge(resource_event_belongs_to(issue_event)) + + ResourceLabelEvent.create!(attrs) end def label_finder diff --git a/lib/gitlab/github_import/importer/events/changed_milestone.rb b/lib/gitlab/github_import/importer/events/changed_milestone.rb index 3164c041dc3..39b92d88b58 100644 --- a/lib/gitlab/github_import/importer/events/changed_milestone.rb +++ b/lib/gitlab/github_import/importer/events/changed_milestone.rb @@ -17,14 +17,15 @@ module Gitlab private def create_event(issue_event) - ResourceMilestoneEvent.create!( - issue_id: issuable_db_id(issue_event), + attrs = { user_id: author_id(issue_event), created_at: issue_event.created_at, milestone_id: project.milestones.find_by_title(issue_event.milestone_title)&.id, action: action(issue_event.event), state: DEFAULT_STATE - ) + }.merge(resource_event_belongs_to(issue_event)) + + ResourceMilestoneEvent.create!(attrs) end def action(event_type) diff --git a/lib/gitlab/github_import/importer/events/changed_reviewer.rb b/lib/gitlab/github_import/importer/events/changed_reviewer.rb new file mode 100644 index 00000000000..17b1fa4ab45 --- /dev/null +++ b/lib/gitlab/github_import/importer/events/changed_reviewer.rb @@ -0,0 +1,54 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Importer + module Events + class ChangedReviewer < BaseImporter + def execute(issue_event) + requested_reviewer_id = author_id(issue_event, author_key: :requested_reviewer) + review_requester_id = author_id(issue_event, author_key: :review_requester) + + note_body = parse_body(issue_event, requested_reviewer_id) + + create_note(issue_event, note_body, review_requester_id) + end + + private + + def create_note(issue_event, note_body, review_requester_id) + Note.create!( + system: true, + noteable_type: issuable_type(issue_event), + noteable_id: issuable_db_id(issue_event), + project: project, + author_id: review_requester_id, + note: note_body, + system_note_metadata: SystemNoteMetadata.new( + { + action: 'reviewer', + created_at: issue_event.created_at, + updated_at: issue_event.created_at + } + ), + created_at: issue_event.created_at, + updated_at: issue_event.created_at + ) + end + + def parse_body(issue_event, requested_reviewer_id) + requested_reviewer = User.find(requested_reviewer_id).to_reference + + if issue_event.event == 'review_request_removed' + "#{SystemNotes::IssuablesService.issuable_events[:review_request_removed]}" \ + " #{requested_reviewer}" + else + "#{SystemNotes::IssuablesService.issuable_events[:review_requested]}" \ + " #{requested_reviewer}" + end + end + end + end + end + end +end diff --git a/lib/gitlab/github_import/importer/events/closed.rb b/lib/gitlab/github_import/importer/events/closed.rb index ca8730d0f27..58d9dbf826c 100644 --- a/lib/gitlab/github_import/importer/events/closed.rb +++ b/lib/gitlab/github_import/importer/events/closed.rb @@ -17,7 +17,7 @@ module Gitlab project_id: project.id, author_id: author_id(issue_event), action: 'closed', - target_type: Issue.name, + target_type: issuable_type(issue_event), target_id: issuable_db_id(issue_event), created_at: issue_event.created_at, updated_at: issue_event.created_at @@ -25,15 +25,16 @@ module Gitlab end def create_state_event(issue_event) - ResourceStateEvent.create!( + attrs = { user_id: author_id(issue_event), - issue_id: issuable_db_id(issue_event), source_commit: issue_event.commit_id, state: 'closed', close_after_error_tracking_resolve: false, close_auto_resolve_prometheus_alert: false, created_at: issue_event.created_at - ) + }.merge(resource_event_belongs_to(issue_event)) + + ResourceStateEvent.create!(attrs) end end end diff --git a/lib/gitlab/github_import/importer/events/cross_referenced.rb b/lib/gitlab/github_import/importer/events/cross_referenced.rb index 89fc1bdeb09..b56ae186d3c 100644 --- a/lib/gitlab/github_import/importer/events/cross_referenced.rb +++ b/lib/gitlab/github_import/importer/events/cross_referenced.rb @@ -33,7 +33,7 @@ module Gitlab def create_note(issue_event, note_body, user_id) Note.create!( system: true, - noteable_type: Issue.name, + noteable_type: issuable_type(issue_event), noteable_id: issuable_db_id(issue_event), project: project, author_id: user_id, diff --git a/lib/gitlab/github_import/importer/events/renamed.rb b/lib/gitlab/github_import/importer/events/renamed.rb index 96d112b04c6..fb9e08116ba 100644 --- a/lib/gitlab/github_import/importer/events/renamed.rb +++ b/lib/gitlab/github_import/importer/events/renamed.rb @@ -14,7 +14,7 @@ module Gitlab def note_params(issue_event) { noteable_id: issuable_db_id(issue_event), - noteable_type: Issue.name, + noteable_type: issuable_type(issue_event), project_id: project.id, author_id: author_id(issue_event), note: parse_body(issue_event), diff --git a/lib/gitlab/github_import/importer/events/reopened.rb b/lib/gitlab/github_import/importer/events/reopened.rb index b75344bf817..8abeba0777d 100644 --- a/lib/gitlab/github_import/importer/events/reopened.rb +++ b/lib/gitlab/github_import/importer/events/reopened.rb @@ -17,7 +17,7 @@ module Gitlab project_id: project.id, author_id: author_id(issue_event), action: 'reopened', - target_type: Issue.name, + target_type: issuable_type(issue_event), target_id: issuable_db_id(issue_event), created_at: issue_event.created_at, updated_at: issue_event.created_at @@ -25,12 +25,13 @@ module Gitlab end def create_state_event(issue_event) - ResourceStateEvent.create!( + attrs = { user_id: author_id(issue_event), - issue_id: issuable_db_id(issue_event), state: 'reopened', created_at: issue_event.created_at - ) + }.merge(resource_event_belongs_to(issue_event)) + + ResourceStateEvent.create!(attrs) end end end diff --git a/lib/gitlab/github_import/importer/issue_event_importer.rb b/lib/gitlab/github_import/importer/issue_event_importer.rb index ef456e56ee1..80749aae93c 100644 --- a/lib/gitlab/github_import/importer/issue_event_importer.rb +++ b/lib/gitlab/github_import/importer/issue_event_importer.rb @@ -15,11 +15,7 @@ module Gitlab @client = client end - # TODO: Add MergeRequest events support - # https://gitlab.com/groups/gitlab-org/-/epics/7673 def execute - return if issue_event.issuable_type == 'MergeRequest' - importer = event_importer_class(issue_event) if importer importer.new(project, client).execute(issue_event) @@ -49,6 +45,8 @@ module Gitlab Gitlab::GithubImport::Importer::Events::CrossReferenced when 'assigned', 'unassigned' Gitlab::GithubImport::Importer::Events::ChangedAssignee + when 'review_requested', 'review_request_removed' + Gitlab::GithubImport::Importer::Events::ChangedReviewer end end end diff --git a/lib/gitlab/github_import/importer/protected_branch_importer.rb b/lib/gitlab/github_import/importer/protected_branch_importer.rb new file mode 100644 index 00000000000..16215fdce8e --- /dev/null +++ b/lib/gitlab/github_import/importer/protected_branch_importer.rb @@ -0,0 +1,48 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Importer + class ProtectedBranchImporter + attr_reader :protected_branch, :project, :client + + # protected_branch - An instance of + # `Gitlab::GithubImport::Representation::ProtectedBranch`. + # project - An instance of `Project` + # client - An instance of `Gitlab::GithubImport::Client` + def initialize(protected_branch, project, client) + @protected_branch = protected_branch + @project = project + @client = client + end + + def execute + # The creator of the project is always allowed to create protected + # branches, so we skip the authorization check in this service class. + ProtectedBranches::CreateService + .new(project, project.creator, params) + .execute(skip_authorization: true) + end + + private + + def params + { + name: protected_branch.id, + push_access_levels_attributes: [{ access_level: Gitlab::Access::MAINTAINER }], + merge_access_levels_attributes: [{ access_level: Gitlab::Access::MAINTAINER }], + allow_force_push: allow_force_push? + } + end + + def allow_force_push? + if ProtectedBranch.protected?(project, protected_branch.id) + ProtectedBranch.allow_force_push?(project, protected_branch.id) && protected_branch.allow_force_pushes + else + protected_branch.allow_force_pushes + end + end + end + end + end +end diff --git a/lib/gitlab/github_import/importer/protected_branches_importer.rb b/lib/gitlab/github_import/importer/protected_branches_importer.rb new file mode 100644 index 00000000000..b5be823d5ab --- /dev/null +++ b/lib/gitlab/github_import/importer/protected_branches_importer.rb @@ -0,0 +1,52 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Importer + class ProtectedBranchesImporter + include ParallelScheduling + + # The method that will be called for traversing through all the objects to + # import, yielding them to the supplied block. + def each_object_to_import + repo = project.import_source + + protected_branches = client.branches(repo).select { |branch| branch.protection&.enabled } + protected_branches.each do |protected_branch| + object = client.branch_protection(repo, protected_branch.name) + next if object.nil? || already_imported?(object) + + yield object + + Gitlab::GithubImport::ObjectCounter.increment(project, object_type, :fetched) + mark_as_imported(object) + end + end + + def importer_class + ProtectedBranchImporter + end + + def representation_class + Gitlab::GithubImport::Representation::ProtectedBranch + end + + def sidekiq_worker_class + ImportProtectedBranchWorker + end + + def object_type + :protected_branch + end + + def collection_method + :protected_branches + end + + def id_for_already_imported_cache(protected_branch) + protected_branch.name + end + end + end + end +end diff --git a/lib/gitlab/github_import/importer/release_attachments_importer.rb b/lib/gitlab/github_import/importer/release_attachments_importer.rb new file mode 100644 index 00000000000..6419851623c --- /dev/null +++ b/lib/gitlab/github_import/importer/release_attachments_importer.rb @@ -0,0 +1,58 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Importer + class ReleaseAttachmentsImporter + attr_reader :release_db_id, :release_description, :project + + # release - An instance of `ReleaseAttachments`. + # project - An instance of `Project`. + # client - An instance of `Gitlab::GithubImport::Client`. + def initialize(release_attachments, project, _client = nil) + @release_db_id = release_attachments.release_db_id + @release_description = release_attachments.description + @project = project + end + + def execute + attachment_urls = MarkdownText.fetch_attachment_urls(release_description) + new_description = attachment_urls.reduce(release_description) do |description, url| + new_url = download_attachment(url) + description.gsub(url, new_url) + end + + Release.find(release_db_id).update_column(:description, new_description) + end + + private + + # in: github attachment markdown url + # out: gitlab attachment markdown url + def download_attachment(markdown_url) + url = extract_url_from_markdown(markdown_url) + name_prefix = extract_name_from_markdown(markdown_url) + + downloader = ::Gitlab::GithubImport::AttachmentsDownloader.new(url) + file = downloader.perform + uploader = UploadService.new(project, file, FileUploader).execute + "#{name_prefix}(#{uploader.to_h[:url]})" + ensure + downloader&.delete + end + + # in: "" + # out: https://user-images.githubusercontent.com/.. + def extract_url_from_markdown(text) + text.match(%r{https://.*\)$}).to_a[0].chop + end + + # in: "" + # out: ![image-icon] + def extract_name_from_markdown(text) + text.match(%r{^!?\[.*\]}).to_a[0] + end + end + end + end +end diff --git a/lib/gitlab/github_import/importer/releases_attachments_importer.rb b/lib/gitlab/github_import/importer/releases_attachments_importer.rb new file mode 100644 index 00000000000..7221c802d83 --- /dev/null +++ b/lib/gitlab/github_import/importer/releases_attachments_importer.rb @@ -0,0 +1,59 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Importer + class ReleasesAttachmentsImporter + include ParallelScheduling + + BATCH_SIZE = 100 + + # The method that will be called for traversing through all the objects to + # import, yielding them to the supplied block. + def each_object_to_import + project.releases.select(:id, :description).each_batch(of: BATCH_SIZE, column: :id) do |batch| + batch.each do |release| + next if already_imported?(release) + + Gitlab::GithubImport::ObjectCounter.increment(project, object_type, :fetched) + + yield release + + # We mark the object as imported immediately so we don't end up + # scheduling it multiple times. + mark_as_imported(release) + end + end + end + + def representation_class + Representation::ReleaseAttachments + end + + def importer_class + ReleaseAttachmentsImporter + end + + def sidekiq_worker_class + ImportReleaseAttachmentsWorker + end + + def collection_method + :release_attachments + end + + def object_type + :release_attachment + end + + def id_for_already_imported_cache(release) + release.id + end + + def object_representation(object) + representation_class.from_db_record(object) + end + end + end + end +end diff --git a/lib/gitlab/github_import/importer/repository_importer.rb b/lib/gitlab/github_import/importer/repository_importer.rb index aba4729e9c8..708768a60cf 100644 --- a/lib/gitlab/github_import/importer/repository_importer.rb +++ b/lib/gitlab/github_import/importer/repository_importer.rb @@ -17,7 +17,7 @@ module Gitlab # Returns true if we should import the wiki for the project. # rubocop: disable CodeReuse/ActiveRecord def import_wiki? - client_repository&.has_wiki && + client_repository[:has_wiki] && !project.wiki_repository_exists? && Gitlab::GitalyClient::RemoteService.exists?(wiki_url) end @@ -86,7 +86,7 @@ module Gitlab private def default_branch - client_repository&.default_branch + client_repository[:default_branch] end def client_repository diff --git a/lib/gitlab/github_import/importer/single_endpoint_issue_events_importer.rb b/lib/gitlab/github_import/importer/single_endpoint_issue_events_importer.rb index 8e4015acbbc..8a9ddfc6ec0 100644 --- a/lib/gitlab/github_import/importer/single_endpoint_issue_events_importer.rb +++ b/lib/gitlab/github_import/importer/single_endpoint_issue_events_importer.rb @@ -7,7 +7,7 @@ module Gitlab include ParallelScheduling include SingleEndpointNotesImporting - PROCESSED_PAGE_CACHE_KEY = 'issues/%{issue_iid}/%{collection}' + PROCESSED_PAGE_CACHE_KEY = 'issues/%{issuable_iid}/%{collection}' BATCH_SIZE = 100 def initialize(project, client, parallel: true) @@ -27,12 +27,20 @@ module Gitlab Gitlab::GithubImport::ObjectCounter.increment(project, object_type, :fetched) - associated.issue = { 'number' => parent_record.iid } + pull_request = parent_record.is_a? MergeRequest + associated.issue = { 'number' => parent_record.iid, 'pull_request' => pull_request } yield(associated) mark_as_imported(associated) end + # In Github Issues and MergeRequests uses the same API to get their events. + # Even more - they have commonly uniq iid + def each_associated_page(&block) + issues_collection.each_batch(of: BATCH_SIZE, column: :iid) { |batch| process_batch(batch, &block) } + merge_requests_collection.each_batch(of: BATCH_SIZE, column: :iid) { |batch| process_batch(batch, &block) } + end + def importer_class IssueEventImporter end @@ -53,16 +61,20 @@ module Gitlab :issue_timeline end - def parent_collection + def issues_collection project.issues.where.not(iid: already_imported_parents).select(:id, :iid) # rubocop: disable CodeReuse/ActiveRecord end + def merge_requests_collection + project.merge_requests.where.not(iid: already_imported_parents).select(:id, :iid) # rubocop: disable CodeReuse/ActiveRecord + end + def parent_imported_cache_key "github-importer/issues/#{collection_method}/already-imported/#{project.id}" end - def page_counter_id(issue) - PROCESSED_PAGE_CACHE_KEY % { issue_iid: issue.iid, collection: collection_method } + def page_counter_id(issuable) + PROCESSED_PAGE_CACHE_KEY % { issuable_iid: issuable.iid, collection: collection_method } end def id_for_already_imported_cache(event) @@ -74,10 +86,10 @@ module Gitlab end # Cross-referenced events on Github doesn't have id. - def compose_associated_id!(issue, event) + def compose_associated_id!(issuable, event) return if event.event != 'cross-referenced' - event.id = "cross-reference##{issue.id}-in-#{event.source.issue.id}" + event.id = "cross-reference##{issuable.iid}-in-#{event.source.issue.id}" end end end |