Add latest changes from gitlab-org/gitlab@14-2-stable-eev14.2.0-rc42

18 files changed, 58 insertions, 10 deletions

diff --git a/app/workers/gitlab/github_import/advance_stage_worker.rb b/app/workers/gitlab/github_import/advance_stage_worker.rb
index f25296f0461..06f0ef623c2 100644
--- a/app/workers/gitlab/github_import/advance_stage_worker.rb
+++ b/app/workers/gitlab/github_import/advance_stage_worker.rb
@@ -9,6 +9,8 @@ module Gitlab
     class AdvanceStageWorker # rubocop:disable Scalability/IdempotentWorker
       include ApplicationWorker
 
+      data_consistency :always
+
       sidekiq_options retry: 3
       include ::Gitlab::Import::AdvanceStage
 
diff --git a/app/workers/gitlab/github_import/import_pull_request_merged_by_worker.rb b/app/workers/gitlab/github_import/import_pull_request_merged_by_worker.rb
index 91dab3470d9..cce179542c7 100644
--- a/app/workers/gitlab/github_import/import_pull_request_merged_by_worker.rb
+++ b/app/workers/gitlab/github_import/import_pull_request_merged_by_worker.rb
@@ -6,6 +6,7 @@ module Gitlab
       include ObjectImporter
 
       tags :exclude_from_kubernetes
+      worker_resource_boundary :cpu
 
       def representation_class
         Gitlab::GithubImport::Representation::PullRequest
diff --git a/app/workers/gitlab/github_import/import_pull_request_review_worker.rb b/app/workers/gitlab/github_import/import_pull_request_review_worker.rb
index de10fe40589..8796d6392df 100644
--- a/app/workers/gitlab/github_import/import_pull_request_review_worker.rb
+++ b/app/workers/gitlab/github_import/import_pull_request_review_worker.rb
@@ -6,6 +6,7 @@ module Gitlab
       include ObjectImporter
 
       tags :exclude_from_kubernetes
+      worker_resource_boundary :cpu
 
       def representation_class
         Gitlab::GithubImport::Representation::PullRequestReview
diff --git a/app/workers/gitlab/github_import/refresh_import_jid_worker.rb b/app/workers/gitlab/github_import/refresh_import_jid_worker.rb
index 1c769921ab3..2b9fb26d53a 100644
--- a/app/workers/gitlab/github_import/refresh_import_jid_worker.rb
+++ b/app/workers/gitlab/github_import/refresh_import_jid_worker.rb
@@ -5,6 +5,8 @@ module Gitlab
     class RefreshImportJidWorker # rubocop:disable Scalability/IdempotentWorker
       include ApplicationWorker
 
+      data_consistency :always
+
       sidekiq_options retry: 3
       include GithubImport::Queue
 
diff --git a/app/workers/gitlab/github_import/stage/finish_import_worker.rb b/app/workers/gitlab/github_import/stage/finish_import_worker.rb
index f909d7e2f34..006b79dbff4 100644
--- a/app/workers/gitlab/github_import/stage/finish_import_worker.rb
+++ b/app/workers/gitlab/github_import/stage/finish_import_worker.rb
@@ -6,6 +6,8 @@ module Gitlab
       class FinishImportWorker # rubocop:disable Scalability/IdempotentWorker
         include ApplicationWorker
 
+        data_consistency :always
+
         sidekiq_options retry: 3
         include GithubImport::Queue
         include StageMethods
diff --git a/app/workers/gitlab/github_import/stage/import_base_data_worker.rb b/app/workers/gitlab/github_import/stage/import_base_data_worker.rb
index 7ca23ecad20..715c39caf42 100644
--- a/app/workers/gitlab/github_import/stage/import_base_data_worker.rb
+++ b/app/workers/gitlab/github_import/stage/import_base_data_worker.rb
@@ -6,6 +6,8 @@ module Gitlab
       class ImportBaseDataWorker # rubocop:disable Scalability/IdempotentWorker
         include ApplicationWorker
 
+        data_consistency :always
+
         sidekiq_options retry: 3
         include GithubImport::Queue
         include StageMethods
diff --git a/app/workers/gitlab/github_import/stage/import_issues_and_diff_notes_worker.rb b/app/workers/gitlab/github_import/stage/import_issues_and_diff_notes_worker.rb
index d66698277b0..c33836e20d1 100644
--- a/app/workers/gitlab/github_import/stage/import_issues_and_diff_notes_worker.rb
+++ b/app/workers/gitlab/github_import/stage/import_issues_and_diff_notes_worker.rb
@@ -6,6 +6,8 @@ module Gitlab
       class ImportIssuesAndDiffNotesWorker # rubocop:disable Scalability/IdempotentWorker
         include ApplicationWorker
 
+        data_consistency :always
+
         sidekiq_options retry: 3
         include GithubImport::Queue
         include StageMethods
diff --git a/app/workers/gitlab/github_import/stage/import_lfs_objects_worker.rb b/app/workers/gitlab/github_import/stage/import_lfs_objects_worker.rb
index 2a66a08d534..f6f5687130f 100644
--- a/app/workers/gitlab/github_import/stage/import_lfs_objects_worker.rb
+++ b/app/workers/gitlab/github_import/stage/import_lfs_objects_worker.rb
@@ -6,6 +6,8 @@ module Gitlab
       class ImportLfsObjectsWorker # rubocop:disable Scalability/IdempotentWorker
         include ApplicationWorker
 
+        data_consistency :always
+
         sidekiq_options retry: 3
         include GithubImport::Queue
         include StageMethods
diff --git a/app/workers/gitlab/github_import/stage/import_notes_worker.rb b/app/workers/gitlab/github_import/stage/import_notes_worker.rb
index 873e389fca6..0160145ffe2 100644
--- a/app/workers/gitlab/github_import/stage/import_notes_worker.rb
+++ b/app/workers/gitlab/github_import/stage/import_notes_worker.rb
@@ -6,6 +6,8 @@ module Gitlab
       class ImportNotesWorker # rubocop:disable Scalability/IdempotentWorker
         include ApplicationWorker
 
+        data_consistency :always
+
         sidekiq_options retry: 3
         include GithubImport::Queue
         include StageMethods
diff --git a/app/workers/gitlab/github_import/stage/import_pull_requests_merged_by_worker.rb b/app/workers/gitlab/github_import/stage/import_pull_requests_merged_by_worker.rb
index 5743648680d..7d83fe288da 100644
--- a/app/workers/gitlab/github_import/stage/import_pull_requests_merged_by_worker.rb
+++ b/app/workers/gitlab/github_import/stage/import_pull_requests_merged_by_worker.rb
@@ -6,6 +6,8 @@ module Gitlab
       class ImportPullRequestsMergedByWorker # rubocop:disable Scalability/IdempotentWorker
         include ApplicationWorker
 
+        data_consistency :always
+
         sidekiq_options retry: 3
         include GithubImport::Queue
         include StageMethods
diff --git a/app/workers/gitlab/github_import/stage/import_pull_requests_reviews_worker.rb b/app/workers/gitlab/github_import/stage/import_pull_requests_reviews_worker.rb
index 532d550f190..ea3b89efd22 100644
--- a/app/workers/gitlab/github_import/stage/import_pull_requests_reviews_worker.rb
+++ b/app/workers/gitlab/github_import/stage/import_pull_requests_reviews_worker.rb
@@ -6,6 +6,8 @@ module Gitlab
       class ImportPullRequestsReviewsWorker # rubocop:disable Scalability/IdempotentWorker
         include ApplicationWorker
 
+        data_consistency :always
+
         sidekiq_options retry: 3
         include GithubImport::Queue
         include StageMethods
diff --git a/app/workers/gitlab/github_import/stage/import_pull_requests_worker.rb b/app/workers/gitlab/github_import/stage/import_pull_requests_worker.rb
index 5755aea21ce..d76d36531d1 100644
--- a/app/workers/gitlab/github_import/stage/import_pull_requests_worker.rb
+++ b/app/workers/gitlab/github_import/stage/import_pull_requests_worker.rb
@@ -6,6 +6,8 @@ module Gitlab
       class ImportPullRequestsWorker # rubocop:disable Scalability/IdempotentWorker
         include ApplicationWorker
 
+        data_consistency :always
+
         sidekiq_options retry: 3
         include GithubImport::Queue
         include StageMethods
diff --git a/app/workers/gitlab/github_import/stage/import_repository_worker.rb b/app/workers/gitlab/github_import/stage/import_repository_worker.rb
index e113563ce8b..227b7c304b0 100644
--- a/app/workers/gitlab/github_import/stage/import_repository_worker.rb
+++ b/app/workers/gitlab/github_import/stage/import_repository_worker.rb
@@ -6,6 +6,8 @@ module Gitlab
       class ImportRepositoryWorker # rubocop:disable Scalability/IdempotentWorker
         include ApplicationWorker
 
+        data_consistency :always
+
         sidekiq_options retry: 3
         include GithubImport::Queue
         include StageMethods
@@ -26,7 +28,7 @@ module Gitlab
           info(project.id, message: "starting importer", importer: 'Importer::RepositoryImporter')
           importer = Importer::RepositoryImporter.new(project, client)
 
-          return unless importer.execute
+          importer.execute
 
           counter.increment
 
@@ -39,6 +41,10 @@ module Gitlab
             'The number of imported GitHub repositories'
           )
         end
+
+        def abort_on_failure
+          true
+        end
       end
     end
   end
diff --git a/app/workers/gitlab/import/stuck_import_job.rb b/app/workers/gitlab/import/stuck_import_job.rb
index 57fb3baf2b5..efbea7d8133 100644
--- a/app/workers/gitlab/import/stuck_import_job.rb
+++ b/app/workers/gitlab/import/stuck_import_job.rb
@@ -5,12 +5,13 @@ module Gitlab
     module StuckImportJob
       extend ActiveSupport::Concern
 
+      StuckImportJobError = Class.new(StandardError)
+
       IMPORT_JOBS_EXPIRATION = 24.hours.seconds.to_i
 
       included do
         include ApplicationWorker
 
-        sidekiq_options retry: 3
         # rubocop:disable Scalability/CronWorkerContext
         # This worker updates several import states inline and does not schedule
         # other jobs. So no context needed
@@ -35,9 +36,9 @@ module Gitlab
       end
 
       def mark_imports_without_jid_as_failed!
-        enqueued_import_states_without_jid.each do |import_state|
-          import_state.mark_as_failed(error_message)
-        end.size
+        enqueued_import_states_without_jid
+          .each(&method(:mark_as_failed))
+          .size
       end
 
       def mark_imports_with_jid_as_failed!
@@ -59,9 +60,20 @@ module Gitlab
           job_ids: completed_import_state_jids
         )
 
-        completed_import_states.each do |import_state|
-          import_state.mark_as_failed(error_message)
-        end.size
+        completed_import_states
+          .each(&method(:mark_as_failed))
+          .size
+      end
+
+      def mark_as_failed(import_state)
+        raise StuckImportJobError, error_message
+      rescue StuckImportJobError => e
+        Gitlab::Import::ImportFailureService.track(
+          import_state: import_state,
+          exception: e,
+          error_source: self.class.name,
+          fail_import: true
+        )
       end
 
       def enqueued_import_states
diff --git a/app/workers/gitlab/jira_import/advance_stage_worker.rb b/app/workers/gitlab/jira_import/advance_stage_worker.rb
index 6387054d448..5fae7caf791 100644
--- a/app/workers/gitlab/jira_import/advance_stage_worker.rb
+++ b/app/workers/gitlab/jira_import/advance_stage_worker.rb
@@ -5,6 +5,8 @@ module Gitlab
     class AdvanceStageWorker # rubocop:disable Scalability/IdempotentWorker
       include ApplicationWorker
 
+      data_consistency :always
+
       sidekiq_options retry: 3
       include QueueOptions
       include ::Gitlab::Import::AdvanceStage
diff --git a/app/workers/gitlab/jira_import/import_issue_worker.rb b/app/workers/gitlab/jira_import/import_issue_worker.rb
index 98bde2218c2..eabe7328b92 100644
--- a/app/workers/gitlab/jira_import/import_issue_worker.rb
+++ b/app/workers/gitlab/jira_import/import_issue_worker.rb
@@ -5,6 +5,8 @@ module Gitlab
     class ImportIssueWorker # rubocop:disable Scalability/IdempotentWorker
       include ApplicationWorker
 
+      data_consistency :always
+
       sidekiq_options retry: 3
       include NotifyUponDeath
       include Gitlab::JiraImport::QueueOptions
@@ -52,7 +54,7 @@ module Gitlab
 
         label_link_attrs << build_label_attrs(issue_id, import_label_id.to_i)
 
-        Gitlab::Database.bulk_insert(LabelLink.table_name, label_link_attrs) # rubocop:disable Gitlab/BulkInsert
+        Gitlab::Database.main.bulk_insert(LabelLink.table_name, label_link_attrs) # rubocop:disable Gitlab/BulkInsert
       end
 
       def assign_issue(project_id, issue_id, assignee_ids)
@@ -60,7 +62,7 @@ module Gitlab
 
         assignee_attrs = assignee_ids.map { |user_id| { issue_id: issue_id, user_id: user_id } }
 
-        Gitlab::Database.bulk_insert(IssueAssignee.table_name, assignee_attrs) # rubocop:disable Gitlab/BulkInsert
+        Gitlab::Database.main.bulk_insert(IssueAssignee.table_name, assignee_attrs) # rubocop:disable Gitlab/BulkInsert
       end
 
       def build_label_attrs(issue_id, label_id)
diff --git a/app/workers/gitlab/jira_import/stage/start_import_worker.rb b/app/workers/gitlab/jira_import/stage/start_import_worker.rb
index e327ced8c65..e0de3ee169e 100644
--- a/app/workers/gitlab/jira_import/stage/start_import_worker.rb
+++ b/app/workers/gitlab/jira_import/stage/start_import_worker.rb
@@ -6,6 +6,8 @@ module Gitlab
       class StartImportWorker # rubocop:disable Scalability/IdempotentWorker
         include ApplicationWorker
 
+        data_consistency :always
+
         sidekiq_options retry: 3
         include ProjectStartImport
         include ProjectImportOptions
diff --git a/app/workers/gitlab/phabricator_import/import_tasks_worker.rb b/app/workers/gitlab/phabricator_import/import_tasks_worker.rb
index 867a12fbac2..f650681fc2f 100644
--- a/app/workers/gitlab/phabricator_import/import_tasks_worker.rb
+++ b/app/workers/gitlab/phabricator_import/import_tasks_worker.rb
@@ -4,6 +4,8 @@ module Gitlab
     class ImportTasksWorker < BaseWorker # rubocop:disable Scalability/IdempotentWorker
       include ApplicationWorker
 
+      data_consistency :always
+
       sidekiq_options retry: 3
       include ProjectImportOptions # This marks the project as failed after too many tries