From 8b573c94895dc0ac0e1d9d59cf3e8745e8b539ca Mon Sep 17 00:00:00 2001 From: GitLab Bot Date: Thu, 17 Dec 2020 11:59:07 +0000 Subject: Add latest changes from gitlab-org/gitlab@13-7-stable-ee --- .../gitlab/github_import/advance_stage_worker.rb | 2 ++ .../import_pull_request_merged_by_worker.rb | 25 +++++++++++++++ .../import_pull_request_review_worker.rb | 25 +++++++++++++++ .../github_import/import_pull_request_worker.rb | 2 +- .../github_import/stage/finish_import_worker.rb | 9 ++++-- .../github_import/stage/import_base_data_worker.rb | 1 + .../stage/import_issues_and_diff_notes_worker.rb | 1 + .../stage/import_lfs_objects_worker.rb | 2 ++ .../github_import/stage/import_notes_worker.rb | 1 + .../stage/import_pull_requests_merged_by_worker.rb | 29 +++++++++++++++++ .../stage/import_pull_requests_reviews_worker.rb | 36 ++++++++++++++++++++++ .../stage/import_pull_requests_worker.rb | 3 +- .../stage/import_repository_worker.rb | 1 + 13 files changed, 132 insertions(+), 5 deletions(-) create mode 100644 app/workers/gitlab/github_import/import_pull_request_merged_by_worker.rb create mode 100644 app/workers/gitlab/github_import/import_pull_request_review_worker.rb create mode 100644 app/workers/gitlab/github_import/stage/import_pull_requests_merged_by_worker.rb create mode 100644 app/workers/gitlab/github_import/stage/import_pull_requests_reviews_worker.rb (limited to 'app/workers/gitlab') diff --git a/app/workers/gitlab/github_import/advance_stage_worker.rb b/app/workers/gitlab/github_import/advance_stage_worker.rb index 834c2f7791c..af406b32415 100644 --- a/app/workers/gitlab/github_import/advance_stage_worker.rb +++ b/app/workers/gitlab/github_import/advance_stage_worker.rb @@ -16,6 +16,8 @@ module Gitlab # The known importer stages and their corresponding Sidekiq workers. STAGES = { + pull_requests_merged_by: Stage::ImportPullRequestsMergedByWorker, + pull_request_reviews: Stage::ImportPullRequestsReviewsWorker, issues_and_diff_notes: Stage::ImportIssuesAndDiffNotesWorker, notes: Stage::ImportNotesWorker, lfs_objects: Stage::ImportLfsObjectsWorker, diff --git a/app/workers/gitlab/github_import/import_pull_request_merged_by_worker.rb b/app/workers/gitlab/github_import/import_pull_request_merged_by_worker.rb new file mode 100644 index 00000000000..79ef917bbc5 --- /dev/null +++ b/app/workers/gitlab/github_import/import_pull_request_merged_by_worker.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + class ImportPullRequestMergedByWorker # rubocop:disable Scalability/IdempotentWorker + include ObjectImporter + + def representation_class + Gitlab::GithubImport::Representation::PullRequest + end + + def importer_class + Importer::PullRequestMergedByImporter + end + + def counter_name + :github_importer_imported_pull_requests_merged_by + end + + def counter_description + 'The number of imported GitHub pull requests merged by' + end + end + end +end diff --git a/app/workers/gitlab/github_import/import_pull_request_review_worker.rb b/app/workers/gitlab/github_import/import_pull_request_review_worker.rb new file mode 100644 index 00000000000..b8516fb8670 --- /dev/null +++ b/app/workers/gitlab/github_import/import_pull_request_review_worker.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + class ImportPullRequestReviewWorker # rubocop:disable Scalability/IdempotentWorker + include ObjectImporter + + def representation_class + Gitlab::GithubImport::Representation::PullRequestReview + end + + def importer_class + Importer::PullRequestReviewImporter + end + + def counter_name + :github_importer_imported_pull_request_reviews + end + + def counter_description + 'The number of imported GitHub pull request reviews' + end + end + end +end diff --git a/app/workers/gitlab/github_import/import_pull_request_worker.rb b/app/workers/gitlab/github_import/import_pull_request_worker.rb index ec806ad170b..9560874f247 100644 --- a/app/workers/gitlab/github_import/import_pull_request_worker.rb +++ b/app/workers/gitlab/github_import/import_pull_request_worker.rb @@ -6,7 +6,7 @@ module Gitlab include ObjectImporter def representation_class - Representation::PullRequest + Gitlab::GithubImport::Representation::PullRequest end def importer_class diff --git a/app/workers/gitlab/github_import/stage/finish_import_worker.rb b/app/workers/gitlab/github_import/stage/finish_import_worker.rb index 73699a74a4a..058e1a0853d 100644 --- a/app/workers/gitlab/github_import/stage/finish_import_worker.rb +++ b/app/workers/gitlab/github_import/stage/finish_import_worker.rb @@ -20,12 +20,15 @@ module Gitlab def report_import_time(project) duration = Time.zone.now - project.created_at - path = project.full_path - histogram.observe({ project: path }, duration) + histogram.observe({ project: project.full_path }, duration) counter.increment - logger.info("GitHub importer finished for #{path} in #{duration.round(2)} seconds") + info( + project.id, + message: "GitHub project import finished", + duration_s: duration.round(2) + ) end def histogram diff --git a/app/workers/gitlab/github_import/stage/import_base_data_worker.rb b/app/workers/gitlab/github_import/stage/import_base_data_worker.rb index 11c2a2ac9b4..202bb335ca1 100644 --- a/app/workers/gitlab/github_import/stage/import_base_data_worker.rb +++ b/app/workers/gitlab/github_import/stage/import_base_data_worker.rb @@ -20,6 +20,7 @@ module Gitlab # project - An instance of Project. def import(client, project) IMPORTERS.each do |klass| + info(project.id, message: "starting importer", importer: klass.name) klass.new(project, client).execute end diff --git a/app/workers/gitlab/github_import/stage/import_issues_and_diff_notes_worker.rb b/app/workers/gitlab/github_import/stage/import_issues_and_diff_notes_worker.rb index 68b6e159fa4..486057804b4 100644 --- a/app/workers/gitlab/github_import/stage/import_issues_and_diff_notes_worker.rb +++ b/app/workers/gitlab/github_import/stage/import_issues_and_diff_notes_worker.rb @@ -19,6 +19,7 @@ module Gitlab # project - An instance of Project. def import(client, project) waiters = IMPORTERS.each_with_object({}) do |klass, hash| + info(project.id, message: "starting importer", importer: klass.name) waiter = klass.new(project, client).execute hash[waiter.key] = waiter.jobs_remaining end diff --git a/app/workers/gitlab/github_import/stage/import_lfs_objects_worker.rb b/app/workers/gitlab/github_import/stage/import_lfs_objects_worker.rb index a19df399969..de2a7f9fc29 100644 --- a/app/workers/gitlab/github_import/stage/import_lfs_objects_worker.rb +++ b/app/workers/gitlab/github_import/stage/import_lfs_objects_worker.rb @@ -16,6 +16,8 @@ module Gitlab # project - An instance of Project. def import(project) + info(project.id, message: "starting importer", importer: 'Importer::LfsObjectsImporter') + waiter = Importer::LfsObjectsImporter .new(project, nil) .execute diff --git a/app/workers/gitlab/github_import/stage/import_notes_worker.rb b/app/workers/gitlab/github_import/stage/import_notes_worker.rb index 49b9821cd45..e1da26a9d48 100644 --- a/app/workers/gitlab/github_import/stage/import_notes_worker.rb +++ b/app/workers/gitlab/github_import/stage/import_notes_worker.rb @@ -11,6 +11,7 @@ module Gitlab # client - An instance of Gitlab::GithubImport::Client. # project - An instance of Project. def import(client, project) + info(project.id, message: "starting importer", importer: 'Importer::NotesImporter') waiter = Importer::NotesImporter .new(project, client) .execute diff --git a/app/workers/gitlab/github_import/stage/import_pull_requests_merged_by_worker.rb b/app/workers/gitlab/github_import/stage/import_pull_requests_merged_by_worker.rb new file mode 100644 index 00000000000..3e15c346659 --- /dev/null +++ b/app/workers/gitlab/github_import/stage/import_pull_requests_merged_by_worker.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Stage + class ImportPullRequestsMergedByWorker # rubocop:disable Scalability/IdempotentWorker + include ApplicationWorker + include GithubImport::Queue + include StageMethods + + # client - An instance of Gitlab::GithubImport::Client. + # project - An instance of Project. + def import(client, project) + waiter = Importer::PullRequestsMergedByImporter + .new(project, client) + .execute + + project.import_state.refresh_jid_expiration + + AdvanceStageWorker.perform_async( + project.id, + { waiter.key => waiter.jobs_remaining }, + :pull_request_reviews + ) + end + end + end + end +end diff --git a/app/workers/gitlab/github_import/stage/import_pull_requests_reviews_worker.rb b/app/workers/gitlab/github_import/stage/import_pull_requests_reviews_worker.rb new file mode 100644 index 00000000000..0809d0b7c29 --- /dev/null +++ b/app/workers/gitlab/github_import/stage/import_pull_requests_reviews_worker.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Stage + class ImportPullRequestsReviewsWorker # rubocop:disable Scalability/IdempotentWorker + include ApplicationWorker + include GithubImport::Queue + include StageMethods + + # client - An instance of Gitlab::GithubImport::Client. + # project - An instance of Project. + def import(client, project) + waiter = + if Feature.enabled?(:github_import_pull_request_reviews, project, default_enabled: true) + waiter = Importer::PullRequestsReviewsImporter + .new(project, client) + .execute + + project.import_state.refresh_jid_expiration + + waiter + else + JobWaiter.new + end + + AdvanceStageWorker.perform_async( + project.id, + { waiter.key => waiter.jobs_remaining }, + :issues_and_diff_notes + ) + end + end + end + end +end diff --git a/app/workers/gitlab/github_import/stage/import_pull_requests_worker.rb b/app/workers/gitlab/github_import/stage/import_pull_requests_worker.rb index 3299db5653b..bf2defa6326 100644 --- a/app/workers/gitlab/github_import/stage/import_pull_requests_worker.rb +++ b/app/workers/gitlab/github_import/stage/import_pull_requests_worker.rb @@ -11,6 +11,7 @@ module Gitlab # client - An instance of Gitlab::GithubImport::Client. # project - An instance of Project. def import(client, project) + info(project.id, message: "starting importer", importer: 'Importer::PullRequestsImporter') waiter = Importer::PullRequestsImporter .new(project, client) .execute @@ -20,7 +21,7 @@ module Gitlab AdvanceStageWorker.perform_async( project.id, { waiter.key => waiter.jobs_remaining }, - :issues_and_diff_notes + :pull_requests_merged_by ) end end diff --git a/app/workers/gitlab/github_import/stage/import_repository_worker.rb b/app/workers/gitlab/github_import/stage/import_repository_worker.rb index cb9ef1cd198..3338f7e58c0 100644 --- a/app/workers/gitlab/github_import/stage/import_repository_worker.rb +++ b/app/workers/gitlab/github_import/stage/import_repository_worker.rb @@ -21,6 +21,7 @@ module Gitlab # expiration time. RefreshImportJidWorker.perform_in_the_future(project.id, jid) + info(project.id, message: "starting importer", importer: 'Importer::RepositoryImporter') importer = Importer::RepositoryImporter.new(project, client) return unless importer.execute -- cgit v1.2.3