diff options
Diffstat (limited to 'lib/gitlab/github_import')
14 files changed, 138 insertions, 8 deletions
diff --git a/lib/gitlab/github_import/importer/diff_notes_importer.rb b/lib/gitlab/github_import/importer/diff_notes_importer.rb index 966f12c5c2f..49cbc8f7a42 100644 --- a/lib/gitlab/github_import/importer/diff_notes_importer.rb +++ b/lib/gitlab/github_import/importer/diff_notes_importer.rb @@ -22,6 +22,10 @@ module Gitlab :pull_requests_comments end + def object_type + :diff_note + end + def id_for_already_imported_cache(note) note.id end diff --git a/lib/gitlab/github_import/importer/issues_importer.rb b/lib/gitlab/github_import/importer/issues_importer.rb index ac6d0666b3a..6cc1a61b332 100644 --- a/lib/gitlab/github_import/importer/issues_importer.rb +++ b/lib/gitlab/github_import/importer/issues_importer.rb @@ -18,6 +18,10 @@ module Gitlab ImportIssueWorker end + def object_type + :issue + end + def collection_method :issues end diff --git a/lib/gitlab/github_import/importer/lfs_objects_importer.rb b/lib/gitlab/github_import/importer/lfs_objects_importer.rb index c74a7706117..40248ecbd31 100644 --- a/lib/gitlab/github_import/importer/lfs_objects_importer.rb +++ b/lib/gitlab/github_import/importer/lfs_objects_importer.rb @@ -18,6 +18,10 @@ module Gitlab ImportLfsObjectWorker end + def object_type + :lfs_object + end + def collection_method :lfs_objects end @@ -26,6 +30,8 @@ module Gitlab lfs_objects = Projects::LfsPointers::LfsObjectDownloadListService.new(project).execute lfs_objects.each do |object| + Gitlab::GithubImport::ObjectCounter.increment(project, object_type, :fetched) + yield object end rescue StandardError => e diff --git a/lib/gitlab/github_import/importer/notes_importer.rb b/lib/gitlab/github_import/importer/notes_importer.rb index 5aec760ea5f..ca1d7d60515 100644 --- a/lib/gitlab/github_import/importer/notes_importer.rb +++ b/lib/gitlab/github_import/importer/notes_importer.rb @@ -18,6 +18,10 @@ module Gitlab ImportNoteWorker end + def object_type + :note + end + def collection_method :issues_comments end diff --git a/lib/gitlab/github_import/importer/pull_request_merged_by_importer.rb b/lib/gitlab/github_import/importer/pull_request_merged_by_importer.rb index 8173fdd5e3e..640914acf4d 100644 --- a/lib/gitlab/github_import/importer/pull_request_merged_by_importer.rb +++ b/lib/gitlab/github_import/importer/pull_request_merged_by_importer.rb @@ -43,7 +43,7 @@ module Gitlab def missing_author_note s_("GitHubImporter|*Merged by: %{author} at %{timestamp}*") % { - author: pull_request.merged_by.login, + author: pull_request.merged_by&.login || 'ghost', timestamp: pull_request.merged_at } end diff --git a/lib/gitlab/github_import/importer/pull_request_review_importer.rb b/lib/gitlab/github_import/importer/pull_request_review_importer.rb index f476ee13392..dd5b7c93ced 100644 --- a/lib/gitlab/github_import/importer/pull_request_review_importer.rb +++ b/lib/gitlab/github_import/importer/pull_request_review_importer.rb @@ -69,8 +69,8 @@ module Gitlab author_id: author_id, note: note, system: false, - created_at: review.submitted_at, - updated_at: review.submitted_at + created_at: submitted_at, + updated_at: submitted_at }.merge(extra) end @@ -80,8 +80,8 @@ module Gitlab approval_attribues = { merge_request_id: merge_request.id, user_id: user_id, - created_at: review.submitted_at, - updated_at: review.submitted_at + created_at: submitted_at, + updated_at: submitted_at } result = ::Approval.insert( @@ -105,6 +105,10 @@ module Gitlab Note.create!(attributes) end + + def submitted_at + @submitted_at ||= (review.submitted_at || merge_request.updated_at) + end end end end diff --git a/lib/gitlab/github_import/importer/pull_requests_importer.rb b/lib/gitlab/github_import/importer/pull_requests_importer.rb index 28cd3f802a2..b2f099761b1 100644 --- a/lib/gitlab/github_import/importer/pull_requests_importer.rb +++ b/lib/gitlab/github_import/importer/pull_requests_importer.rb @@ -22,6 +22,10 @@ module Gitlab pr.number end + def object_type + :pull_request + end + def each_object_to_import super do |pr| update_repository if update_repository?(pr) diff --git a/lib/gitlab/github_import/importer/pull_requests_merged_by_importer.rb b/lib/gitlab/github_import/importer/pull_requests_merged_by_importer.rb index 94472cd341e..287e0ea7f7f 100644 --- a/lib/gitlab/github_import/importer/pull_requests_merged_by_importer.rb +++ b/lib/gitlab/github_import/importer/pull_requests_merged_by_importer.rb @@ -22,6 +22,10 @@ module Gitlab :pull_requests_merged_by end + def object_type + :pull_request_merged_by + end + def id_for_already_imported_cache(merge_request) merge_request.id end @@ -30,6 +34,8 @@ module Gitlab project.merge_requests.with_state(:merged).find_each do |merge_request| next if already_imported?(merge_request) + Gitlab::GithubImport::ObjectCounter.increment(project, object_type, :fetched) + pull_request = client.pull_request(project.import_source, merge_request.iid) yield(pull_request) diff --git a/lib/gitlab/github_import/importer/pull_requests_reviews_importer.rb b/lib/gitlab/github_import/importer/pull_requests_reviews_importer.rb index 809a518d13a..e389acbf877 100644 --- a/lib/gitlab/github_import/importer/pull_requests_reviews_importer.rb +++ b/lib/gitlab/github_import/importer/pull_requests_reviews_importer.rb @@ -29,6 +29,10 @@ module Gitlab :pull_request_reviews end + def object_type + :pull_request_review + end + def id_for_already_imported_cache(review) review.id end @@ -57,6 +61,8 @@ module Gitlab project.merge_requests.find_each do |merge_request| next if already_imported?(merge_request) + Gitlab::GithubImport::ObjectCounter.increment(project, object_type, :fetched) + client .pull_request_reviews(project.import_source, merge_request.iid) .each do |review| @@ -81,6 +87,8 @@ module Gitlab page.objects.each do |review| next if already_imported?(review) + Gitlab::GithubImport::ObjectCounter.increment(project, object_type, :fetched) + review.merge_request_id = merge_request.id yield(review) diff --git a/lib/gitlab/github_import/markdown_text.rb b/lib/gitlab/github_import/markdown_text.rb index e5f4dabe42d..0b1c221bbec 100644 --- a/lib/gitlab/github_import/markdown_text.rb +++ b/lib/gitlab/github_import/markdown_text.rb @@ -13,7 +13,7 @@ module Gitlab # author - An instance of `Gitlab::GithubImport::Representation::User` # exists - Boolean that indicates the user exists in the GitLab database. def initialize(text, author, exists = false) - @text = text + @text = text.to_s @author = author @exists = exists end diff --git a/lib/gitlab/github_import/object_counter.rb b/lib/gitlab/github_import/object_counter.rb new file mode 100644 index 00000000000..e4835504c2d --- /dev/null +++ b/lib/gitlab/github_import/object_counter.rb @@ -0,0 +1,84 @@ +# frozen_string_literal: true + +# Count objects fetched or imported from Github. +module Gitlab + module GithubImport + class ObjectCounter + OPERATIONS = %w[fetched imported].freeze + PROJECT_COUNTER_LIST_KEY = 'github-importer/object-counters-list/%{project}/%{operation}' + PROJECT_COUNTER_KEY = 'github-importer/object-counter/%{project}/%{operation}/%{object_type}' + + GLOBAL_COUNTER_KEY = 'github_importer_%{operation}_%{object_type}' + GLOBAL_COUNTER_DESCRIPTION = 'The number of %{operation} Github %{object_type}' + + CACHING = Gitlab::Cache::Import::Caching + + class << self + def increment(project, object_type, operation) + validate_operation!(operation) + + increment_project_counter(project, object_type, operation) + increment_global_counter(object_type, operation) + end + + def summary(project) + OPERATIONS.each_with_object({}) do |operation, result| + result[operation] = {} + + CACHING + .values_from_set(counter_list_key(project, operation)) + .sort + .each do |counter| + object_type = counter.split('/').last + result[operation][object_type] = CACHING.read_integer(counter) + end + end + end + + private + + # Global counters are long lived, in Prometheus, + # and it's used to report the health of the Github Importer + # in the Grafana Dashboard + # https://dashboards.gitlab.net/d/2zgM_rImz/github-importer?orgId=1 + def increment_global_counter(object_type, operation) + key = GLOBAL_COUNTER_KEY % { + operation: operation, + object_type: object_type + } + description = GLOBAL_COUNTER_DESCRIPTION % { + operation: operation, + object_type: object_type.to_s.humanize + } + + Gitlab::Metrics.counter(key.to_sym, description).increment + end + + # Project counters are short lived, in Redis, + # and it's used to report how successful a project + # import was with the #summary method. + def increment_project_counter(project, object_type, operation) + counter_key = PROJECT_COUNTER_KEY % { project: project.id, operation: operation, object_type: object_type } + + add_counter_to_list(project, operation, counter_key) + + CACHING.increment(counter_key) + end + + def add_counter_to_list(project, operation, key) + CACHING.set_add(counter_list_key(project, operation), key) + end + + def counter_list_key(project, operation) + PROJECT_COUNTER_LIST_KEY % { project: project.id, operation: operation } + end + + def validate_operation!(operation) + unless operation.to_s.presence_in(OPERATIONS) + raise ArgumentError, "Operation must be #{OPERATIONS.join(' or ')}" + end + end + end + end + end +end diff --git a/lib/gitlab/github_import/parallel_scheduling.rb b/lib/gitlab/github_import/parallel_scheduling.rb index 92f9e8a646d..4598429d568 100644 --- a/lib/gitlab/github_import/parallel_scheduling.rb +++ b/lib/gitlab/github_import/parallel_scheduling.rb @@ -103,6 +103,8 @@ module Gitlab page.objects.each do |object| next if already_imported?(object) + Gitlab::GithubImport::ObjectCounter.increment(project, object_type, :fetched) + yield object # We mark the object as imported immediately so we don't end up @@ -129,6 +131,10 @@ module Gitlab Gitlab::Cache::Import::Caching.set_add(already_imported_cache_key, id) end + def object_type + raise NotImplementedError + end + # Returns the ID to use for the cache used for checking if an object has # already been imported or not. # diff --git a/lib/gitlab/github_import/representation/pull_request_review.rb b/lib/gitlab/github_import/representation/pull_request_review.rb index 3205259a1ed..08b3160fc4c 100644 --- a/lib/gitlab/github_import/representation/pull_request_review.rb +++ b/lib/gitlab/github_import/representation/pull_request_review.rb @@ -29,7 +29,7 @@ module Gitlab hash = Representation.symbolize_hash(raw_hash) hash[:author] &&= Representation::User.from_json_hash(hash[:author]) - hash[:submitted_at] = Time.parse(hash[:submitted_at]).in_time_zone + hash[:submitted_at] = Time.parse(hash[:submitted_at]).in_time_zone if hash[:submitted_at].present? new(hash) end diff --git a/lib/gitlab/github_import/user_finder.rb b/lib/gitlab/github_import/user_finder.rb index 8d584415202..058cd1ebd57 100644 --- a/lib/gitlab/github_import/user_finder.rb +++ b/lib/gitlab/github_import/user_finder.rb @@ -138,7 +138,7 @@ module Gitlab # rubocop: disable CodeReuse/ActiveRecord def query_id_for_github_id(id) - User.for_github_id(id).pluck(:id).first + User.by_provider_and_extern_uid(:github, id).select(:id).first&.id end # rubocop: enable CodeReuse/ActiveRecord |