From b76ae638462ab0f673e5915986070518dd3f9ad3 Mon Sep 17 00:00:00 2001 From: GitLab Bot Date: Thu, 19 Aug 2021 09:08:42 +0000 Subject: Add latest changes from gitlab-org/gitlab@14-2-stable-ee --- .../gitlab/github_import/advance_stage_worker.rb | 2 ++ .../import_pull_request_merged_by_worker.rb | 1 + .../import_pull_request_review_worker.rb | 1 + .../github_import/refresh_import_jid_worker.rb | 2 ++ .../github_import/stage/finish_import_worker.rb | 2 ++ .../github_import/stage/import_base_data_worker.rb | 2 ++ .../stage/import_issues_and_diff_notes_worker.rb | 2 ++ .../stage/import_lfs_objects_worker.rb | 2 ++ .../github_import/stage/import_notes_worker.rb | 2 ++ .../stage/import_pull_requests_merged_by_worker.rb | 2 ++ .../stage/import_pull_requests_reviews_worker.rb | 2 ++ .../stage/import_pull_requests_worker.rb | 2 ++ .../stage/import_repository_worker.rb | 8 ++++++- app/workers/gitlab/import/stuck_import_job.rb | 26 ++++++++++++++++------ .../gitlab/jira_import/advance_stage_worker.rb | 2 ++ .../gitlab/jira_import/import_issue_worker.rb | 6 +++-- .../jira_import/stage/start_import_worker.rb | 2 ++ .../phabricator_import/import_tasks_worker.rb | 2 ++ 18 files changed, 58 insertions(+), 10 deletions(-) (limited to 'app/workers/gitlab') diff --git a/app/workers/gitlab/github_import/advance_stage_worker.rb b/app/workers/gitlab/github_import/advance_stage_worker.rb index f25296f0461..06f0ef623c2 100644 --- a/app/workers/gitlab/github_import/advance_stage_worker.rb +++ b/app/workers/gitlab/github_import/advance_stage_worker.rb @@ -9,6 +9,8 @@ module Gitlab class AdvanceStageWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 include ::Gitlab::Import::AdvanceStage diff --git a/app/workers/gitlab/github_import/import_pull_request_merged_by_worker.rb b/app/workers/gitlab/github_import/import_pull_request_merged_by_worker.rb index 91dab3470d9..cce179542c7 100644 --- a/app/workers/gitlab/github_import/import_pull_request_merged_by_worker.rb +++ b/app/workers/gitlab/github_import/import_pull_request_merged_by_worker.rb @@ -6,6 +6,7 @@ module Gitlab include ObjectImporter tags :exclude_from_kubernetes + worker_resource_boundary :cpu def representation_class Gitlab::GithubImport::Representation::PullRequest diff --git a/app/workers/gitlab/github_import/import_pull_request_review_worker.rb b/app/workers/gitlab/github_import/import_pull_request_review_worker.rb index de10fe40589..8796d6392df 100644 --- a/app/workers/gitlab/github_import/import_pull_request_review_worker.rb +++ b/app/workers/gitlab/github_import/import_pull_request_review_worker.rb @@ -6,6 +6,7 @@ module Gitlab include ObjectImporter tags :exclude_from_kubernetes + worker_resource_boundary :cpu def representation_class Gitlab::GithubImport::Representation::PullRequestReview diff --git a/app/workers/gitlab/github_import/refresh_import_jid_worker.rb b/app/workers/gitlab/github_import/refresh_import_jid_worker.rb index 1c769921ab3..2b9fb26d53a 100644 --- a/app/workers/gitlab/github_import/refresh_import_jid_worker.rb +++ b/app/workers/gitlab/github_import/refresh_import_jid_worker.rb @@ -5,6 +5,8 @@ module Gitlab class RefreshImportJidWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 include GithubImport::Queue diff --git a/app/workers/gitlab/github_import/stage/finish_import_worker.rb b/app/workers/gitlab/github_import/stage/finish_import_worker.rb index f909d7e2f34..006b79dbff4 100644 --- a/app/workers/gitlab/github_import/stage/finish_import_worker.rb +++ b/app/workers/gitlab/github_import/stage/finish_import_worker.rb @@ -6,6 +6,8 @@ module Gitlab class FinishImportWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 include GithubImport::Queue include StageMethods diff --git a/app/workers/gitlab/github_import/stage/import_base_data_worker.rb b/app/workers/gitlab/github_import/stage/import_base_data_worker.rb index 7ca23ecad20..715c39caf42 100644 --- a/app/workers/gitlab/github_import/stage/import_base_data_worker.rb +++ b/app/workers/gitlab/github_import/stage/import_base_data_worker.rb @@ -6,6 +6,8 @@ module Gitlab class ImportBaseDataWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 include GithubImport::Queue include StageMethods diff --git a/app/workers/gitlab/github_import/stage/import_issues_and_diff_notes_worker.rb b/app/workers/gitlab/github_import/stage/import_issues_and_diff_notes_worker.rb index d66698277b0..c33836e20d1 100644 --- a/app/workers/gitlab/github_import/stage/import_issues_and_diff_notes_worker.rb +++ b/app/workers/gitlab/github_import/stage/import_issues_and_diff_notes_worker.rb @@ -6,6 +6,8 @@ module Gitlab class ImportIssuesAndDiffNotesWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 include GithubImport::Queue include StageMethods diff --git a/app/workers/gitlab/github_import/stage/import_lfs_objects_worker.rb b/app/workers/gitlab/github_import/stage/import_lfs_objects_worker.rb index 2a66a08d534..f6f5687130f 100644 --- a/app/workers/gitlab/github_import/stage/import_lfs_objects_worker.rb +++ b/app/workers/gitlab/github_import/stage/import_lfs_objects_worker.rb @@ -6,6 +6,8 @@ module Gitlab class ImportLfsObjectsWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 include GithubImport::Queue include StageMethods diff --git a/app/workers/gitlab/github_import/stage/import_notes_worker.rb b/app/workers/gitlab/github_import/stage/import_notes_worker.rb index 873e389fca6..0160145ffe2 100644 --- a/app/workers/gitlab/github_import/stage/import_notes_worker.rb +++ b/app/workers/gitlab/github_import/stage/import_notes_worker.rb @@ -6,6 +6,8 @@ module Gitlab class ImportNotesWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 include GithubImport::Queue include StageMethods diff --git a/app/workers/gitlab/github_import/stage/import_pull_requests_merged_by_worker.rb b/app/workers/gitlab/github_import/stage/import_pull_requests_merged_by_worker.rb index 5743648680d..7d83fe288da 100644 --- a/app/workers/gitlab/github_import/stage/import_pull_requests_merged_by_worker.rb +++ b/app/workers/gitlab/github_import/stage/import_pull_requests_merged_by_worker.rb @@ -6,6 +6,8 @@ module Gitlab class ImportPullRequestsMergedByWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 include GithubImport::Queue include StageMethods diff --git a/app/workers/gitlab/github_import/stage/import_pull_requests_reviews_worker.rb b/app/workers/gitlab/github_import/stage/import_pull_requests_reviews_worker.rb index 532d550f190..ea3b89efd22 100644 --- a/app/workers/gitlab/github_import/stage/import_pull_requests_reviews_worker.rb +++ b/app/workers/gitlab/github_import/stage/import_pull_requests_reviews_worker.rb @@ -6,6 +6,8 @@ module Gitlab class ImportPullRequestsReviewsWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 include GithubImport::Queue include StageMethods diff --git a/app/workers/gitlab/github_import/stage/import_pull_requests_worker.rb b/app/workers/gitlab/github_import/stage/import_pull_requests_worker.rb index 5755aea21ce..d76d36531d1 100644 --- a/app/workers/gitlab/github_import/stage/import_pull_requests_worker.rb +++ b/app/workers/gitlab/github_import/stage/import_pull_requests_worker.rb @@ -6,6 +6,8 @@ module Gitlab class ImportPullRequestsWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 include GithubImport::Queue include StageMethods diff --git a/app/workers/gitlab/github_import/stage/import_repository_worker.rb b/app/workers/gitlab/github_import/stage/import_repository_worker.rb index e113563ce8b..227b7c304b0 100644 --- a/app/workers/gitlab/github_import/stage/import_repository_worker.rb +++ b/app/workers/gitlab/github_import/stage/import_repository_worker.rb @@ -6,6 +6,8 @@ module Gitlab class ImportRepositoryWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 include GithubImport::Queue include StageMethods @@ -26,7 +28,7 @@ module Gitlab info(project.id, message: "starting importer", importer: 'Importer::RepositoryImporter') importer = Importer::RepositoryImporter.new(project, client) - return unless importer.execute + importer.execute counter.increment @@ -39,6 +41,10 @@ module Gitlab 'The number of imported GitHub repositories' ) end + + def abort_on_failure + true + end end end end diff --git a/app/workers/gitlab/import/stuck_import_job.rb b/app/workers/gitlab/import/stuck_import_job.rb index 57fb3baf2b5..efbea7d8133 100644 --- a/app/workers/gitlab/import/stuck_import_job.rb +++ b/app/workers/gitlab/import/stuck_import_job.rb @@ -5,12 +5,13 @@ module Gitlab module StuckImportJob extend ActiveSupport::Concern + StuckImportJobError = Class.new(StandardError) + IMPORT_JOBS_EXPIRATION = 24.hours.seconds.to_i included do include ApplicationWorker - sidekiq_options retry: 3 # rubocop:disable Scalability/CronWorkerContext # This worker updates several import states inline and does not schedule # other jobs. So no context needed @@ -35,9 +36,9 @@ module Gitlab end def mark_imports_without_jid_as_failed! - enqueued_import_states_without_jid.each do |import_state| - import_state.mark_as_failed(error_message) - end.size + enqueued_import_states_without_jid + .each(&method(:mark_as_failed)) + .size end def mark_imports_with_jid_as_failed! @@ -59,9 +60,20 @@ module Gitlab job_ids: completed_import_state_jids ) - completed_import_states.each do |import_state| - import_state.mark_as_failed(error_message) - end.size + completed_import_states + .each(&method(:mark_as_failed)) + .size + end + + def mark_as_failed(import_state) + raise StuckImportJobError, error_message + rescue StuckImportJobError => e + Gitlab::Import::ImportFailureService.track( + import_state: import_state, + exception: e, + error_source: self.class.name, + fail_import: true + ) end def enqueued_import_states diff --git a/app/workers/gitlab/jira_import/advance_stage_worker.rb b/app/workers/gitlab/jira_import/advance_stage_worker.rb index 6387054d448..5fae7caf791 100644 --- a/app/workers/gitlab/jira_import/advance_stage_worker.rb +++ b/app/workers/gitlab/jira_import/advance_stage_worker.rb @@ -5,6 +5,8 @@ module Gitlab class AdvanceStageWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 include QueueOptions include ::Gitlab::Import::AdvanceStage diff --git a/app/workers/gitlab/jira_import/import_issue_worker.rb b/app/workers/gitlab/jira_import/import_issue_worker.rb index 98bde2218c2..eabe7328b92 100644 --- a/app/workers/gitlab/jira_import/import_issue_worker.rb +++ b/app/workers/gitlab/jira_import/import_issue_worker.rb @@ -5,6 +5,8 @@ module Gitlab class ImportIssueWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 include NotifyUponDeath include Gitlab::JiraImport::QueueOptions @@ -52,7 +54,7 @@ module Gitlab label_link_attrs << build_label_attrs(issue_id, import_label_id.to_i) - Gitlab::Database.bulk_insert(LabelLink.table_name, label_link_attrs) # rubocop:disable Gitlab/BulkInsert + Gitlab::Database.main.bulk_insert(LabelLink.table_name, label_link_attrs) # rubocop:disable Gitlab/BulkInsert end def assign_issue(project_id, issue_id, assignee_ids) @@ -60,7 +62,7 @@ module Gitlab assignee_attrs = assignee_ids.map { |user_id| { issue_id: issue_id, user_id: user_id } } - Gitlab::Database.bulk_insert(IssueAssignee.table_name, assignee_attrs) # rubocop:disable Gitlab/BulkInsert + Gitlab::Database.main.bulk_insert(IssueAssignee.table_name, assignee_attrs) # rubocop:disable Gitlab/BulkInsert end def build_label_attrs(issue_id, label_id) diff --git a/app/workers/gitlab/jira_import/stage/start_import_worker.rb b/app/workers/gitlab/jira_import/stage/start_import_worker.rb index e327ced8c65..e0de3ee169e 100644 --- a/app/workers/gitlab/jira_import/stage/start_import_worker.rb +++ b/app/workers/gitlab/jira_import/stage/start_import_worker.rb @@ -6,6 +6,8 @@ module Gitlab class StartImportWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 include ProjectStartImport include ProjectImportOptions diff --git a/app/workers/gitlab/phabricator_import/import_tasks_worker.rb b/app/workers/gitlab/phabricator_import/import_tasks_worker.rb index 867a12fbac2..f650681fc2f 100644 --- a/app/workers/gitlab/phabricator_import/import_tasks_worker.rb +++ b/app/workers/gitlab/phabricator_import/import_tasks_worker.rb @@ -4,6 +4,8 @@ module Gitlab class ImportTasksWorker < BaseWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + data_consistency :always + sidekiq_options retry: 3 include ProjectImportOptions # This marks the project as failed after too many tries -- cgit v1.2.3