diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2023-10-19 15:57:54 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2023-10-19 15:57:54 +0300 |
commit | 419c53ec62de6e97a517abd5fdd4cbde3a942a34 (patch) | |
tree | 1f43a548b46bca8a5fb8fe0c31cef1883d49c5b6 /app/workers/concerns | |
parent | 1da20d9135b3ad9e75e65b028bffc921aaf8deb7 (diff) |
Add latest changes from gitlab-org/gitlab@16-5-stable-eev16.5.0-rc42
Diffstat (limited to 'app/workers/concerns')
-rw-r--r-- | app/workers/concerns/auto_devops_queue.rb | 2 | ||||
-rw-r--r-- | app/workers/concerns/chaos_queue.rb | 2 | ||||
-rw-r--r-- | app/workers/concerns/gitlab/github_import/object_importer.rb | 36 | ||||
-rw-r--r-- | app/workers/concerns/gitlab/github_import/queue.rb | 8 | ||||
-rw-r--r-- | app/workers/concerns/gitlab/github_import/stage_methods.rb | 22 | ||||
-rw-r--r-- | app/workers/concerns/limited_capacity/job_tracker.rb | 1 | ||||
-rw-r--r-- | app/workers/concerns/limited_capacity/worker.rb | 36 | ||||
-rw-r--r-- | app/workers/concerns/worker_attributes.rb | 4 |
8 files changed, 32 insertions, 79 deletions
diff --git a/app/workers/concerns/auto_devops_queue.rb b/app/workers/concerns/auto_devops_queue.rb index 61e3c1544bd..cdf429a8be5 100644 --- a/app/workers/concerns/auto_devops_queue.rb +++ b/app/workers/concerns/auto_devops_queue.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true -# + module AutoDevopsQueue extend ActiveSupport::Concern diff --git a/app/workers/concerns/chaos_queue.rb b/app/workers/concerns/chaos_queue.rb index 23e58b5182b..9a3d518dda8 100644 --- a/app/workers/concerns/chaos_queue.rb +++ b/app/workers/concerns/chaos_queue.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true -# + module ChaosQueue extend ActiveSupport::Concern diff --git a/app/workers/concerns/gitlab/github_import/object_importer.rb b/app/workers/concerns/gitlab/github_import/object_importer.rb index e190ced5073..fcc7a96fa2b 100644 --- a/app/workers/concerns/gitlab/github_import/object_importer.rb +++ b/app/workers/concerns/gitlab/github_import/object_importer.rb @@ -10,7 +10,6 @@ module Gitlab included do include ApplicationWorker - sidekiq_options retry: 3 include GithubImport::Queue include ReschedulingMethods @@ -19,11 +18,8 @@ module Gitlab sidekiq_retries_exhausted do |msg| args = msg['args'] - correlation_id = msg['correlation_id'] jid = msg['jid'] - new.perform_failure(args[0], args[1], correlation_id) - # If a job is being exhausted we still want to notify the # Gitlab::Import::AdvanceStageWorker to prevent the entire import from getting stuck if args.length == 3 && (key = args.last) && key.is_a?(String) @@ -64,29 +60,15 @@ module Gitlab rescue NoMethodError => e # This exception will be more useful in development when a new # Representation is created but the developer forgot to add a - # `:github_identifiers` field. + # `#github_identifiers` method. track_and_raise_exception(project, e, fail_import: true) rescue ActiveRecord::RecordInvalid, NotRetriableError => e # We do not raise exception to prevent job retry - failure = track_exception(project, e) - add_identifiers_to_failure(failure, object.github_identifiers) + track_exception(project, e) rescue StandardError => e track_and_raise_exception(project, e) end - # hash - A Hash containing the details of the object to import. - def perform_failure(project_id, hash, correlation_id) - project = Project.find_by_id(project_id) - return unless project - - failure = project.import_failures.failures_by_correlation_id(correlation_id).first - return unless failure - - object = representation_class.from_json_hash(hash) - - add_identifiers_to_failure(failure, object.github_identifiers) - end - def increment_object_counter?(_object) true end @@ -118,16 +100,20 @@ module Gitlab extra.merge( project_id: project_id, importer: importer_class.name, - github_identifiers: github_identifiers + external_identifiers: github_identifiers ) end def track_exception(project, exception, fail_import: false) + external_identifiers = github_identifiers || {} + external_identifiers[:object_type] ||= object_type&.to_s + Gitlab::Import::ImportFailureService.track( project_id: project.id, error_source: importer_class.name, exception: exception, - fail_import: fail_import + fail_import: fail_import, + external_identifiers: external_identifiers ) end @@ -136,12 +122,6 @@ module Gitlab raise(exception) end - - def add_identifiers_to_failure(failure, external_identifiers) - external_identifiers[:object_type] = object_type - - failure.update_column(:external_identifiers, external_identifiers) - end end end end diff --git a/app/workers/concerns/gitlab/github_import/queue.rb b/app/workers/concerns/gitlab/github_import/queue.rb index e7156ac12f8..7cc23dd7c0b 100644 --- a/app/workers/concerns/gitlab/github_import/queue.rb +++ b/app/workers/concerns/gitlab/github_import/queue.rb @@ -15,14 +15,6 @@ module Gitlab # this is better than a project being stuck in the "import" state # forever. sidekiq_options dead: false, retry: 5 - - sidekiq_retries_exhausted do |msg, e| - Gitlab::Import::ImportFailureService.track( - project_id: msg['args'][0], - exception: e, - fail_import: true - ) - end end end end diff --git a/app/workers/concerns/gitlab/github_import/stage_methods.rb b/app/workers/concerns/gitlab/github_import/stage_methods.rb index 75db5589415..80013ff3cd9 100644 --- a/app/workers/concerns/gitlab/github_import/stage_methods.rb +++ b/app/workers/concerns/gitlab/github_import/stage_methods.rb @@ -3,6 +3,21 @@ module Gitlab module GithubImport module StageMethods + extend ActiveSupport::Concern + + included do + include ApplicationWorker + + sidekiq_retries_exhausted do |msg, e| + Gitlab::Import::ImportFailureService.track( + project_id: msg['args'][0], + exception: e, + error_source: self.class.name, + fail_import: true + ) + end + end + # project_id - The ID of the GitLab project to import the data into. def perform(project_id) info(project_id, message: 'starting stage') @@ -29,7 +44,8 @@ module Gitlab project_id: project_id, exception: e, error_source: self.class.name, - fail_import: abort_on_failure + fail_import: false, + metrics: true ) raise(e) @@ -51,10 +67,6 @@ module Gitlab # rubocop: enable CodeReuse/ActiveRecord end - def abort_on_failure - false - end - private def info(project_id, extra = {}) diff --git a/app/workers/concerns/limited_capacity/job_tracker.rb b/app/workers/concerns/limited_capacity/job_tracker.rb index 4b5ce8a01f6..b4d884f914d 100644 --- a/app/workers/concerns/limited_capacity/job_tracker.rb +++ b/app/workers/concerns/limited_capacity/job_tracker.rb @@ -1,4 +1,5 @@ # frozen_string_literal: true + module LimitedCapacity class JobTracker # rubocop:disable Scalability/IdempotentWorker include Gitlab::Utils::StrongMemoize diff --git a/app/workers/concerns/limited_capacity/worker.rb b/app/workers/concerns/limited_capacity/worker.rb index af66d80b3e9..0a79c5c46d5 100644 --- a/app/workers/concerns/limited_capacity/worker.rb +++ b/app/workers/concerns/limited_capacity/worker.rb @@ -1,41 +1,5 @@ # frozen_string_literal: true -# Usage: -# -# Worker that performs the tasks: -# -# class DummyWorker -# include ApplicationWorker -# include LimitedCapacity::Worker -# -# # For each job that raises any error, a worker instance will be disabled -# # until the next schedule-run. -# # If you wish to get around this, exceptions must by handled by the implementer. -# # -# def perform_work(*args) -# end -# -# def remaining_work_count(*args) -# 5 -# end -# -# def max_running_jobs -# 25 -# end -# end -# -# Cron worker to fill the pool of regular workers: -# -# class ScheduleDummyCronWorker -# include ApplicationWorker -# include CronjobQueue -# -# def perform(*args) -# DummyWorker.perform_with_capacity(*args) -# end -# end -# - module LimitedCapacity module Worker extend ActiveSupport::Concern diff --git a/app/workers/concerns/worker_attributes.rb b/app/workers/concerns/worker_attributes.rb index 02eda924b71..cb09aaf1a6a 100644 --- a/app/workers/concerns/worker_attributes.rb +++ b/app/workers/concerns/worker_attributes.rb @@ -155,6 +155,10 @@ module WorkerAttributes ::Gitlab::SidekiqMiddleware::PauseControl::WorkersMap.set_strategy_for(strategy: value, worker: self) end + def get_pause_control + ::Gitlab::SidekiqMiddleware::PauseControl::WorkersMap.strategy_for(worker: self) + end + def get_weight get_class_attribute(:weight) || NAMESPACE_WEIGHTS[queue_namespace] || |