diff options
Diffstat (limited to 'app/workers')
44 files changed, 585 insertions, 210 deletions
diff --git a/app/workers/all_queues.yml b/app/workers/all_queues.yml index a0f6da57f9e..b9168a65764 100644 --- a/app/workers/all_queues.yml +++ b/app/workers/all_queues.yml @@ -147,6 +147,15 @@ :weight: 1 :idempotent: false :tags: [] +- :name: container_repository_delete:container_registry_delete_container_repository + :worker_name: ContainerRegistry::DeleteContainerRepositoryWorker + :feature_category: :container_registry + :has_external_dependencies: false + :urgency: :low + :resource_boundary: :unknown + :weight: 1 + :idempotent: true + :tags: [] - :name: cronjob:admin_email :worker_name: AdminEmailWorker :feature_category: :source_code_management @@ -282,6 +291,15 @@ :weight: 1 :idempotent: false :tags: [] +- :name: cronjob:container_registry_cleanup + :worker_name: ContainerRegistry::CleanupWorker + :feature_category: :container_registry + :has_external_dependencies: false + :urgency: :low + :resource_boundary: :unknown + :weight: 1 + :idempotent: true + :tags: [] - :name: cronjob:container_registry_migration_enqueuer :worker_name: ContainerRegistry::Migration::EnqueuerWorker :feature_category: :container_registry @@ -779,7 +797,7 @@ :tags: [] - :name: cronjob:users_deactivate_dormant_users :worker_name: Users::DeactivateDormantUsersWorker - :feature_category: :utilization + :feature_category: :subscription_cost_management :has_external_dependencies: false :urgency: :low :resource_boundary: :unknown @@ -1146,6 +1164,15 @@ :weight: 1 :idempotent: false :tags: [] +- :name: github_importer:github_import_pull_requests_import_review_request + :worker_name: Gitlab::GithubImport::PullRequests::ImportReviewRequestWorker + :feature_category: :importers + :has_external_dependencies: true + :urgency: :low + :resource_boundary: :cpu + :weight: 1 + :idempotent: false + :tags: [] - :name: github_importer:github_import_refresh_import_jid :worker_name: Gitlab::GithubImport::RefreshImportJidWorker :feature_category: :importers @@ -1245,6 +1272,15 @@ :weight: 1 :idempotent: false :tags: [] +- :name: github_importer:github_import_stage_import_pull_requests_review_requests + :worker_name: Gitlab::GithubImport::Stage::ImportPullRequestsReviewRequestsWorker + :feature_category: :importers + :has_external_dependencies: false + :urgency: :low + :resource_boundary: :unknown + :weight: 1 + :idempotent: false + :tags: [] - :name: github_importer:github_import_stage_import_pull_requests_reviews :worker_name: Gitlab::GithubImport::Stage::ImportPullRequestsReviewsWorker :feature_category: :importers @@ -2694,6 +2730,15 @@ :weight: 1 :idempotent: true :tags: [] +- :name: merge_requests_delete_branch + :worker_name: MergeRequests::DeleteBranchWorker + :feature_category: :source_code_management + :has_external_dependencies: false + :urgency: :high + :resource_boundary: :unknown + :weight: 1 + :idempotent: true + :tags: [] - :name: merge_requests_delete_source_branch :worker_name: MergeRequests::DeleteSourceBranchWorker :feature_category: :source_code_management @@ -3236,7 +3281,7 @@ :tags: [] - :name: update_highest_role :worker_name: UpdateHighestRoleWorker - :feature_category: :utilization + :feature_category: :subscription_usage_reports :has_external_dependencies: false :urgency: :high :resource_boundary: :unknown diff --git a/app/workers/authorized_keys_worker.rb b/app/workers/authorized_keys_worker.rb index 039fe629a61..77f2ed5f495 100644 --- a/app/workers/authorized_keys_worker.rb +++ b/app/workers/authorized_keys_worker.rb @@ -7,8 +7,6 @@ class AuthorizedKeysWorker sidekiq_options retry: 3 - PERMITTED_ACTIONS = %w[add_key remove_key].freeze - feature_category :source_code_management urgency :high weight 2 diff --git a/app/workers/bulk_imports/entity_worker.rb b/app/workers/bulk_imports/entity_worker.rb index ada3210624c..d23d57c33ab 100644 --- a/app/workers/bulk_imports/entity_worker.rb +++ b/app/workers/bulk_imports/entity_worker.rb @@ -12,13 +12,18 @@ module BulkImports worker_has_external_dependencies! def perform(entity_id, current_stage = nil) + @entity = ::BulkImports::Entity.find(entity_id) + if stage_running?(entity_id, current_stage) logger.info( structured_payload( bulk_import_entity_id: entity_id, - bulk_import_id: bulk_import_id(entity_id), + bulk_import_id: entity.bulk_import_id, + bulk_import_entity_type: entity.source_type, + source_full_path: entity.source_full_path, current_stage: current_stage, message: 'Stage running', + source_version: source_version, importer: 'gitlab_migration' ) ) @@ -29,9 +34,12 @@ module BulkImports logger.info( structured_payload( bulk_import_entity_id: entity_id, - bulk_import_id: bulk_import_id(entity_id), + bulk_import_id: entity.bulk_import_id, + bulk_import_entity_type: entity.source_type, + source_full_path: entity.source_full_path, current_stage: current_stage, message: 'Stage starting', + source_version: source_version, importer: 'gitlab_migration' ) ) @@ -44,23 +52,34 @@ module BulkImports ) end rescue StandardError => e - logger.error( - structured_payload( + log_exception(e, + { bulk_import_entity_id: entity_id, - bulk_import_id: bulk_import_id(entity_id), + bulk_import_id: entity.bulk_import_id, + bulk_import_entity_type: entity.source_type, + source_full_path: entity.source_full_path, current_stage: current_stage, - message: e.message, + message: 'Entity failed', + source_version: source_version, importer: 'gitlab_migration' - ) + } ) Gitlab::ErrorTracking.track_exception( - e, bulk_import_entity_id: entity_id, bulk_import_id: bulk_import_id(entity_id), importer: 'gitlab_migration' + e, + bulk_import_entity_id: entity_id, + bulk_import_id: entity.bulk_import_id, + bulk_import_entity_type: entity.source_type, + source_full_path: entity.source_full_path, + source_version: source_version, + importer: 'gitlab_migration' ) end private + attr_reader :entity + def stage_running?(entity_id, stage) return unless stage @@ -71,12 +90,18 @@ module BulkImports BulkImports::Tracker.next_pipeline_trackers_for(entity_id).update(status_event: 'enqueue') end - def bulk_import_id(entity_id) - @bulk_import_id ||= Entity.find(entity_id).bulk_import_id + def source_version + entity.bulk_import.source_version_info.to_s end def logger @logger ||= Gitlab::Import::Logger.build end + + def log_exception(exception, payload) + Gitlab::ExceptionLogFormatter.format!(exception, payload) + + logger.error(structured_payload(payload)) + end end end diff --git a/app/workers/bulk_imports/export_request_worker.rb b/app/workers/bulk_imports/export_request_worker.rb index a57071ddcf1..1a5f6250429 100644 --- a/app/workers/bulk_imports/export_request_worker.rb +++ b/app/workers/bulk_imports/export_request_worker.rb @@ -22,7 +22,19 @@ module BulkImports if e.retriable?(entity) retry_request(e, entity) else - log_export_failure(e, entity) + log_exception(e, + { + bulk_import_entity_id: entity.id, + bulk_import_id: entity.bulk_import_id, + bulk_import_entity_type: entity.source_type, + source_full_path: entity.source_full_path, + message: "Request to export #{entity.source_type} failed", + source_version: entity.bulk_import.source_version_info.to_s, + importer: 'gitlab_migration' + } + ) + + BulkImports::Failure.create(failure_attributes(e, entity)) entity.fail_op! end @@ -41,22 +53,7 @@ module BulkImports ) end - def log_export_failure(exception, entity) - Gitlab::Import::Logger.error( - structured_payload( - log_attributes(exception, entity).merge( - bulk_import_id: entity.bulk_import_id, - bulk_import_entity_type: entity.source_type, - message: "Request to export #{entity.source_type} failed", - importer: 'gitlab_migration' - ) - ) - ) - - BulkImports::Failure.create(log_attributes(exception, entity)) - end - - def log_attributes(exception, entity) + def failure_attributes(exception, entity) { bulk_import_entity_id: entity.id, pipeline_class: 'ExportRequestWorker', @@ -84,15 +81,16 @@ module BulkImports ::GlobalID.parse(response.dig(*query.data_path, 'id')).model_id rescue StandardError => e - Gitlab::Import::Logger.error( - structured_payload( - log_attributes(e, entity).merge( - message: 'Failed to fetch source entity id', - bulk_import_id: entity.bulk_import_id, - bulk_import_entity_type: entity.source_type, - importer: 'gitlab_migration' - ) - ) + log_exception(e, + { + message: 'Failed to fetch source entity id', + bulk_import_entity_id: entity.id, + bulk_import_id: entity.bulk_import_id, + bulk_import_entity_type: entity.source_type, + source_full_path: entity.source_full_path, + source_version: entity.bulk_import.source_version_info.to_s, + importer: 'gitlab_migration' + } ) nil @@ -107,18 +105,29 @@ module BulkImports end def retry_request(exception, entity) - Gitlab::Import::Logger.error( - structured_payload( - log_attributes(exception, entity).merge( - message: 'Retrying export request', - bulk_import_id: entity.bulk_import_id, - bulk_import_entity_type: entity.source_type, - importer: 'gitlab_migration' - ) - ) + log_exception(exception, + { + message: 'Retrying export request', + bulk_import_entity_id: entity.id, + bulk_import_id: entity.bulk_import_id, + bulk_import_entity_type: entity.source_type, + source_full_path: entity.source_full_path, + source_version: entity.bulk_import.source_version_info.to_s, + importer: 'gitlab_migration' + } ) self.class.perform_in(2.seconds, entity.id) end + + def logger + @logger ||= Gitlab::Import::Logger.build + end + + def log_exception(exception, payload) + Gitlab::ExceptionLogFormatter.format!(exception, payload) + + logger.error(structured_payload(payload)) + end end end diff --git a/app/workers/bulk_imports/pipeline_worker.rb b/app/workers/bulk_imports/pipeline_worker.rb index 6d314774cff..5716f6e3f31 100644 --- a/app/workers/bulk_imports/pipeline_worker.rb +++ b/app/workers/bulk_imports/pipeline_worker.rb @@ -17,24 +17,34 @@ module BulkImports .find_by_id(pipeline_tracker_id) if pipeline_tracker.present? + @entity = @pipeline_tracker.entity + logger.info( structured_payload( - bulk_import_entity_id: pipeline_tracker.entity.id, - bulk_import_id: pipeline_tracker.entity.bulk_import_id, + bulk_import_entity_id: entity.id, + bulk_import_id: entity.bulk_import_id, + bulk_import_entity_type: entity.source_type, + source_full_path: entity.source_full_path, pipeline_name: pipeline_tracker.pipeline_name, message: 'Pipeline starting', + source_version: source_version, importer: 'gitlab_migration' ) ) run else + @entity = ::BulkImports::Entity.find(entity_id) + logger.error( structured_payload( bulk_import_entity_id: entity_id, - bulk_import_id: bulk_import_id(entity_id), + bulk_import_id: entity.bulk_import_id, + bulk_import_entity_type: entity.source_type, + source_full_path: entity.source_full_path, pipeline_tracker_id: pipeline_tracker_id, message: 'Unstarted pipeline not found', + source_version: source_version, importer: 'gitlab_migration' ) ) @@ -46,10 +56,10 @@ module BulkImports private - attr_reader :pipeline_tracker + attr_reader :pipeline_tracker, :entity def run - return skip_tracker if pipeline_tracker.entity.failed? + return skip_tracker if entity.failed? raise(Pipeline::ExpiredError, 'Pipeline timeout') if job_timeout? raise(Pipeline::FailedError, "Export from source instance failed: #{export_status.error}") if export_failed? @@ -65,33 +75,39 @@ module BulkImports fail_tracker(e) end - def bulk_import_id(entity_id) - @bulk_import_id ||= Entity.find(entity_id).bulk_import_id + def source_version + entity.bulk_import.source_version_info.to_s end def fail_tracker(exception) pipeline_tracker.update!(status_event: 'fail_op', jid: jid) - logger.error( - structured_payload( - bulk_import_entity_id: pipeline_tracker.entity.id, - bulk_import_id: pipeline_tracker.entity.bulk_import_id, + log_exception(exception, + { + bulk_import_entity_id: entity.id, + bulk_import_id: entity.bulk_import_id, + bulk_import_entity_type: entity.source_type, + source_full_path: entity.source_full_path, pipeline_name: pipeline_tracker.pipeline_name, - message: exception.message, + message: 'Pipeline failed', + source_version: source_version, importer: 'gitlab_migration' - ) + } ) Gitlab::ErrorTracking.track_exception( exception, - bulk_import_entity_id: pipeline_tracker.entity.id, - bulk_import_id: pipeline_tracker.entity.bulk_import_id, + bulk_import_entity_id: entity.id, + bulk_import_id: entity.bulk_import_id, + bulk_import_entity_type: entity.source_type, + source_full_path: entity.source_full_path, pipeline_name: pipeline_tracker.pipeline_name, + source_version: source_version, importer: 'gitlab_migration' ) BulkImports::Failure.create( - bulk_import_entity_id: context.entity.id, + bulk_import_entity_id: entity.id, pipeline_class: pipeline_tracker.pipeline_name, pipeline_step: 'pipeline_worker_run', exception_class: exception.class.to_s, @@ -109,7 +125,7 @@ module BulkImports delay, pipeline_tracker.id, pipeline_tracker.stage, - pipeline_tracker.entity.id + entity.id ) end @@ -128,7 +144,7 @@ module BulkImports def job_timeout? return false unless file_extraction_pipeline? - (Time.zone.now - pipeline_tracker.entity.created_at) > Pipeline::NDJSON_EXPORT_TIMEOUT + (Time.zone.now - entity.created_at) > Pipeline::NDJSON_EXPORT_TIMEOUT end def export_failed? @@ -150,14 +166,17 @@ module BulkImports end def retry_tracker(exception) - logger.error( - structured_payload( - bulk_import_entity_id: pipeline_tracker.entity.id, - bulk_import_id: pipeline_tracker.entity.bulk_import_id, + log_exception(exception, + { + bulk_import_entity_id: entity.id, + bulk_import_id: entity.bulk_import_id, + bulk_import_entity_type: entity.source_type, + source_full_path: entity.source_full_path, pipeline_name: pipeline_tracker.pipeline_name, - message: "Retrying error: #{exception.message}", + message: "Retrying pipeline", + source_version: source_version, importer: 'gitlab_migration' - ) + } ) pipeline_tracker.update!(status_event: 'retry', jid: jid) @@ -168,15 +187,23 @@ module BulkImports def skip_tracker logger.info( structured_payload( - bulk_import_entity_id: pipeline_tracker.entity.id, - bulk_import_id: pipeline_tracker.entity.bulk_import_id, + bulk_import_entity_id: entity.id, + bulk_import_id: entity.bulk_import_id, + bulk_import_entity_type: entity.source_type, + source_full_path: entity.source_full_path, pipeline_name: pipeline_tracker.pipeline_name, message: 'Skipping pipeline due to failed entity', + source_version: source_version, importer: 'gitlab_migration' ) ) pipeline_tracker.update!(status_event: 'skip', jid: jid) end + + def log_exception(exception, payload) + Gitlab::ExceptionLogFormatter.format!(exception, payload) + logger.error(structured_payload(payload)) + end end end diff --git a/app/workers/cluster_configure_istio_worker.rb b/app/workers/cluster_configure_istio_worker.rb index 0def66b7381..8bf723d89f7 100644 --- a/app/workers/cluster_configure_istio_worker.rb +++ b/app/workers/cluster_configure_istio_worker.rb @@ -1,5 +1,8 @@ # frozen_string_literal: true +# DEPRECATED +# +# To be removed by https://gitlab.com/gitlab-org/gitlab/-/issues/366573 class ClusterConfigureIstioWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker @@ -10,9 +13,5 @@ class ClusterConfigureIstioWorker # rubocop:disable Scalability/IdempotentWorker worker_has_external_dependencies! - def perform(cluster_id) - Clusters::Cluster.find_by_id(cluster_id).try do |cluster| - Clusters::Kubernetes::ConfigureIstioIngressService.new(cluster: cluster).execute - end - end + def perform(cluster_id); end end diff --git a/app/workers/cluster_install_app_worker.rb b/app/workers/cluster_install_app_worker.rb index e16e6e9ca71..0c94f8cad6a 100644 --- a/app/workers/cluster_install_app_worker.rb +++ b/app/workers/cluster_install_app_worker.rb @@ -1,5 +1,8 @@ # frozen_string_literal: true +# DEPRECATED +# +# To be removed by https://gitlab.com/gitlab-org/gitlab/-/issues/366573 class ClusterInstallAppWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker @@ -12,9 +15,5 @@ class ClusterInstallAppWorker # rubocop:disable Scalability/IdempotentWorker worker_has_external_dependencies! loggable_arguments 0 - def perform(app_name, app_id) - find_application(app_name, app_id) do |app| - Clusters::Applications::InstallService.new(app).execute - end - end + def perform(app_name, app_id); end end diff --git a/app/workers/cluster_patch_app_worker.rb b/app/workers/cluster_patch_app_worker.rb index bb16cf7a5e6..1ef9dc7f6fe 100644 --- a/app/workers/cluster_patch_app_worker.rb +++ b/app/workers/cluster_patch_app_worker.rb @@ -1,5 +1,8 @@ # frozen_string_literal: true +# DEPRECATED +# +# To be removed by https://gitlab.com/gitlab-org/gitlab/-/issues/366573 class ClusterPatchAppWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker @@ -12,9 +15,5 @@ class ClusterPatchAppWorker # rubocop:disable Scalability/IdempotentWorker worker_has_external_dependencies! loggable_arguments 0 - def perform(app_name, app_id) - find_application(app_name, app_id) do |app| - Clusters::Applications::PatchService.new(app).execute - end - end + def perform(app_name, app_id); end end diff --git a/app/workers/cluster_update_app_worker.rb b/app/workers/cluster_update_app_worker.rb index 97fdec02ba4..7d997c0a293 100644 --- a/app/workers/cluster_update_app_worker.rb +++ b/app/workers/cluster_update_app_worker.rb @@ -1,6 +1,7 @@ # frozen_string_literal: true # Deprecated, to be removed in %14.0 as part of https://gitlab.com/groups/gitlab-org/-/epics/4280 +# Also see https://gitlab.com/gitlab-org/gitlab/-/issues/366573 class ClusterUpdateAppWorker # rubocop:disable Scalability/IdempotentWorker UpdateAlreadyInProgressError = Class.new(StandardError) @@ -16,38 +17,5 @@ class ClusterUpdateAppWorker # rubocop:disable Scalability/IdempotentWorker LEASE_TIMEOUT = 10.minutes.to_i - def perform(app_name, app_id, project_id, scheduled_time) - @app_id = app_id - - try_obtain_lease do - execute(app_name, app_id, project_id, scheduled_time) - end - end - - private - - def execute(app_name, app_id, project_id, scheduled_time) - project = Project.find_by_id(project_id) - return unless project - - find_application(app_name, app_id) do |app| - update_prometheus(app, scheduled_time, project) - end - end - - def update_prometheus(app, scheduled_time, project) - return unless app.managed_prometheus? - return if app.updated_since?(scheduled_time) - return if app.update_in_progress? - - Clusters::Applications::PrometheusUpdateService.new(app, project).execute - end - - def lease_key - @lease_key ||= "#{self.class.name.underscore}-#{@app_id}" - end - - def lease_timeout - LEASE_TIMEOUT - end + def perform(app_name, app_id, project_id, scheduled_time); end end diff --git a/app/workers/cluster_upgrade_app_worker.rb b/app/workers/cluster_upgrade_app_worker.rb index bbe0cb7f0c2..40feee9374d 100644 --- a/app/workers/cluster_upgrade_app_worker.rb +++ b/app/workers/cluster_upgrade_app_worker.rb @@ -1,5 +1,8 @@ # frozen_string_literal: true +# DEPRECATED +# +# To be removed by https://gitlab.com/gitlab-org/gitlab/-/issues/366573 class ClusterUpgradeAppWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker @@ -12,9 +15,5 @@ class ClusterUpgradeAppWorker # rubocop:disable Scalability/IdempotentWorker worker_has_external_dependencies! loggable_arguments 0 - def perform(app_name, app_id) - find_application(app_name, app_id) do |app| - Clusters::Applications::UpgradeService.new(app).execute - end - end + def perform(app_name, app_id); end end diff --git a/app/workers/cluster_wait_for_app_installation_worker.rb b/app/workers/cluster_wait_for_app_installation_worker.rb index 846e4442233..ec291ddeb10 100644 --- a/app/workers/cluster_wait_for_app_installation_worker.rb +++ b/app/workers/cluster_wait_for_app_installation_worker.rb @@ -1,5 +1,8 @@ # frozen_string_literal: true +# DEPRECATED +# +# To be removed by https://gitlab.com/gitlab-org/gitlab/-/issues/366573 class ClusterWaitForAppInstallationWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker @@ -16,9 +19,5 @@ class ClusterWaitForAppInstallationWorker # rubocop:disable Scalability/Idempote worker_resource_boundary :cpu loggable_arguments 0 - def perform(app_name, app_id) - find_application(app_name, app_id) do |app| - Clusters::Applications::CheckInstallationProgressService.new(app).execute - end - end + def perform(app_name, app_id); end end diff --git a/app/workers/cluster_wait_for_app_update_worker.rb b/app/workers/cluster_wait_for_app_update_worker.rb index e96e03ae249..084e8b41a49 100644 --- a/app/workers/cluster_wait_for_app_update_worker.rb +++ b/app/workers/cluster_wait_for_app_update_worker.rb @@ -1,5 +1,8 @@ # frozen_string_literal: true +# DEPRECATED +# +# To be removed by https://gitlab.com/gitlab-org/gitlab/-/issues/366573 class ClusterWaitForAppUpdateWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker @@ -14,9 +17,5 @@ class ClusterWaitForAppUpdateWorker # rubocop:disable Scalability/IdempotentWork loggable_arguments 0 - def perform(app_name, app_id) - find_application(app_name, app_id) do |app| - ::Clusters::Applications::CheckUpgradeProgressService.new(app).execute - end - end + def perform(app_name, app_id); end end diff --git a/app/workers/cluster_wait_for_ingress_ip_address_worker.rb b/app/workers/cluster_wait_for_ingress_ip_address_worker.rb index 561e72562e9..8983942c0fb 100644 --- a/app/workers/cluster_wait_for_ingress_ip_address_worker.rb +++ b/app/workers/cluster_wait_for_ingress_ip_address_worker.rb @@ -1,5 +1,8 @@ # frozen_string_literal: true +# DEPRECATED +# +# To be removed by https://gitlab.com/gitlab-org/gitlab/-/issues/366573 class ClusterWaitForIngressIpAddressWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker @@ -12,9 +15,5 @@ class ClusterWaitForIngressIpAddressWorker # rubocop:disable Scalability/Idempot worker_has_external_dependencies! loggable_arguments 0 - def perform(app_name, app_id) - find_application(app_name, app_id) do |app| - Clusters::Applications::CheckIngressIpAddressService.new(app).execute - end - end + def perform(app_name, app_id); end end diff --git a/app/workers/clusters/applications/deactivate_integration_worker.rb b/app/workers/clusters/applications/deactivate_integration_worker.rb index d1db99d21af..fca05e8ad2e 100644 --- a/app/workers/clusters/applications/deactivate_integration_worker.rb +++ b/app/workers/clusters/applications/deactivate_integration_worker.rb @@ -24,6 +24,8 @@ module Clusters .include_integration(integration_association_name) projects.find_each do |project| + # This use of public_send is safe because we constructed the + # integration_association_name ourselves above. project.public_send(integration_association_name).update!(active: false) # rubocop:disable GitlabSecurity/PublicSend end end diff --git a/app/workers/clusters/applications/wait_for_uninstall_app_worker.rb b/app/workers/clusters/applications/wait_for_uninstall_app_worker.rb index 510ea8e7b17..974d99139a1 100644 --- a/app/workers/clusters/applications/wait_for_uninstall_app_worker.rb +++ b/app/workers/clusters/applications/wait_for_uninstall_app_worker.rb @@ -1,5 +1,8 @@ # frozen_string_literal: true +# DEPRECATED +# +# To be removed by https://gitlab.com/gitlab-org/gitlab/-/issues/366573 module Clusters module Applications class WaitForUninstallAppWorker # rubocop:disable Scalability/IdempotentWorker @@ -18,11 +21,7 @@ module Clusters worker_resource_boundary :cpu loggable_arguments 0 - def perform(app_name, app_id) - find_application(app_name, app_id) do |app| - Clusters::Applications::CheckUninstallProgressService.new(app).execute - end - end + def perform(app_name, app_id); end end end end diff --git a/app/workers/concerns/limited_capacity/job_tracker.rb b/app/workers/concerns/limited_capacity/job_tracker.rb index a1eb4e45027..4b5ce8a01f6 100644 --- a/app/workers/concerns/limited_capacity/job_tracker.rb +++ b/app/workers/concerns/limited_capacity/job_tracker.rb @@ -58,7 +58,7 @@ module LimitedCapacity end def remove_job_keys(redis, keys) - redis.srem(counter_key, keys) + redis.srem?(counter_key, keys) end def with_redis(&block) diff --git a/app/workers/container_registry/cleanup_worker.rb b/app/workers/container_registry/cleanup_worker.rb new file mode 100644 index 00000000000..8350ae3431b --- /dev/null +++ b/app/workers/container_registry/cleanup_worker.rb @@ -0,0 +1,59 @@ +# frozen_string_literal: true + +module ContainerRegistry + class CleanupWorker + include ApplicationWorker + # we don't have any project, user or group context here + include CronjobQueue # rubocop:disable Scalability/CronWorkerContext + + data_consistency :always + idempotent! + + feature_category :container_registry + + STALE_DELETE_THRESHOLD = 30.minutes.freeze + BATCH_SIZE = 200 + + def perform + return unless Feature.enabled?(:container_registry_delete_repository_with_cron_worker) + + log_counts + + reset_stale_deletes + + enqueue_delete_container_repository_jobs if ContainerRepository.delete_scheduled.exists? + end + + private + + def reset_stale_deletes + ContainerRepository.delete_ongoing.each_batch(of: BATCH_SIZE) do |batch| + batch.with_stale_delete_at(STALE_DELETE_THRESHOLD.ago).update_all( + status: :delete_scheduled, + delete_started_at: nil + ) + end + end + + def enqueue_delete_container_repository_jobs + ContainerRegistry::DeleteContainerRepositoryWorker.perform_with_capacity + end + + def log_counts + ::Gitlab::Database::LoadBalancing::Session.current.use_replicas_for_read_queries do + log_extra_metadata_on_done( + :delete_scheduled_container_repositories_count, + ContainerRepository.delete_scheduled.count + ) + log_extra_metadata_on_done( + :stale_delete_container_repositories_count, + stale_delete_container_repositories.count + ) + end + end + + def stale_delete_container_repositories + ContainerRepository.delete_ongoing.with_stale_delete_at(STALE_DELETE_THRESHOLD.ago) + end + end +end diff --git a/app/workers/container_registry/delete_container_repository_worker.rb b/app/workers/container_registry/delete_container_repository_worker.rb new file mode 100644 index 00000000000..1f94b1b9e71 --- /dev/null +++ b/app/workers/container_registry/delete_container_repository_worker.rb @@ -0,0 +1,81 @@ +# frozen_string_literal: true + +module ContainerRegistry + class DeleteContainerRepositoryWorker + include ApplicationWorker + include LimitedCapacity::Worker + include Gitlab::Utils::StrongMemoize + extend ::Gitlab::Utils::Override + + data_consistency :always + queue_namespace :container_repository_delete + feature_category :container_registry + urgency :low + worker_resource_boundary :unknown + idempotent! + + MAX_CAPACITY = 2 + CLEANUP_TAGS_SERVICE_PARAMS = { + 'name_regex_delete' => '.*', + 'container_expiration_policy' => true # to avoid permissions checks + }.freeze + + def perform_work + return unless next_container_repository + + result = delete_tags + log_delete_tags_service_result(next_container_repository, result) + + if result[:status] == :error || next_container_repository.tags_count != 0 + return next_container_repository.set_delete_scheduled_status + end + + next_container_repository.destroy! + rescue StandardError => exception + next_container_repository&.set_delete_scheduled_status + + Gitlab::ErrorTracking.log_exception(exception, class: self.class.name) + end + + def remaining_work_count + ::ContainerRepository.delete_scheduled.limit(max_running_jobs + 1).count + end + + def max_running_jobs + MAX_CAPACITY + end + + private + + def delete_tags + service = Projects::ContainerRepository::CleanupTagsService.new( + container_repository: next_container_repository, + params: CLEANUP_TAGS_SERVICE_PARAMS + ) + service.execute + end + + def next_container_repository + strong_memoize(:next_container_repository) do + ContainerRepository.transaction do + # we don't care about the order + repository = ContainerRepository.next_pending_destruction(order_by: nil) + + repository&.tap(&:set_delete_ongoing_status) + end + end + end + + def log_delete_tags_service_result(container_repository, delete_tags_service_result) + logger.info( + structured_payload( + project_id: container_repository.project_id, + container_repository_id: container_repository.id, + container_repository_path: container_repository.path, + tags_size_before_delete: delete_tags_service_result[:original_size], + deleted_tags_size: delete_tags_service_result[:deleted_size] + ) + ) + end + end +end diff --git a/app/workers/database/batched_background_migration/execution_worker.rb b/app/workers/database/batched_background_migration/execution_worker.rb new file mode 100644 index 00000000000..098153c742f --- /dev/null +++ b/app/workers/database/batched_background_migration/execution_worker.rb @@ -0,0 +1,75 @@ +# frozen_string_literal: true + +module Database + module BatchedBackgroundMigration + class ExecutionWorker # rubocop:disable Scalability/IdempotentWorker + include ExclusiveLeaseGuard + include Gitlab::Utils::StrongMemoize + + INTERVAL_VARIANCE = 5.seconds.freeze + LEASE_TIMEOUT_MULTIPLIER = 3 + + def perform(database_name, migration_id) + self.database_name = database_name + + return unless enabled? + return if shares_db_config? + + Gitlab::Database::SharedModel.using_connection(base_model.connection) do + self.migration = find_migration(migration_id) + + break unless migration + + try_obtain_lease do + run_migration_job if executable_migration? + end + end + end + + private + + attr_accessor :database_name, :migration + + def enabled? + Feature.enabled?(:execute_batched_migrations_on_schedule, type: :ops) + end + + def shares_db_config? + Gitlab::Database.db_config_share_with(base_model.connection_db_config).present? + end + + def base_model + strong_memoize(:base_model) do + Gitlab::Database.database_base_models[database_name] + end + end + + def find_migration(id) + Gitlab::Database::BackgroundMigration::BatchedMigration.find_executable(id, connection: base_model.connection) + end + + def lease_key + @lease_key ||= [ + self.class.name.underscore, + 'database_name', + database_name, + 'table_name', + migration.table_name + ].join(':') + end + + def lease_timeout + migration.interval * LEASE_TIMEOUT_MULTIPLIER + end + + def executable_migration? + migration.active? && migration.interval_elapsed?(variance: INTERVAL_VARIANCE) + end + + def run_migration_job + Gitlab::Database::BackgroundMigration::BatchedMigrationRunner.new(connection: base_model.connection) + .run_migration_job(migration) + end + end + end +end diff --git a/app/workers/database/batched_background_migration/single_database_worker.rb b/app/workers/database/batched_background_migration/single_database_worker.rb index cfbd44ba397..0c7c51d5c0a 100644 --- a/app/workers/database/batched_background_migration/single_database_worker.rb +++ b/app/workers/database/batched_background_migration/single_database_worker.rb @@ -58,14 +58,7 @@ module Database break unless self.class.enabled? && active_migration with_exclusive_lease(active_migration.interval) do - # Now that we have the exclusive lease, reload migration in case another process has changed it. - # This is a temporary solution until we have better concurrency handling around job execution - # - # We also have to disable this cop, because ApplicationRecord aliases reset to reload, but our database - # models don't inherit from ApplicationRecord - active_migration.reload # rubocop:disable Cop/ActiveRecordAssociationReload - - run_active_migration if active_migration.active? && active_migration.interval_elapsed?(variance: INTERVAL_VARIANCE) + run_active_migration end end end @@ -77,7 +70,7 @@ module Database end def run_active_migration - Gitlab::Database::BackgroundMigration::BatchedMigrationRunner.new(connection: base_model.connection).run_migration_job(active_migration) + Database::BatchedBackgroundMigration::ExecutionWorker.new.perform(self.class.tracking_database, active_migration.id) end def base_model diff --git a/app/workers/gitlab/github_import/advance_stage_worker.rb b/app/workers/gitlab/github_import/advance_stage_worker.rb index fdf4ec6f396..a9f645bd634 100644 --- a/app/workers/gitlab/github_import/advance_stage_worker.rb +++ b/app/workers/gitlab/github_import/advance_stage_worker.rb @@ -21,6 +21,7 @@ module Gitlab # The known importer stages and their corresponding Sidekiq workers. STAGES = { pull_requests_merged_by: Stage::ImportPullRequestsMergedByWorker, + pull_request_review_requests: Stage::ImportPullRequestsReviewRequestsWorker, pull_request_reviews: Stage::ImportPullRequestsReviewsWorker, issues_and_diff_notes: Stage::ImportIssuesAndDiffNotesWorker, issue_events: Stage::ImportIssueEventsWorker, diff --git a/app/workers/gitlab/github_import/pull_requests/import_review_request_worker.rb b/app/workers/gitlab/github_import/pull_requests/import_review_request_worker.rb new file mode 100644 index 00000000000..e475a39810d --- /dev/null +++ b/app/workers/gitlab/github_import/pull_requests/import_review_request_worker.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module PullRequests + class ImportReviewRequestWorker # rubocop:disable Scalability/IdempotentWorker + include ObjectImporter + + worker_resource_boundary :cpu + + def representation_class + Gitlab::GithubImport::Representation::PullRequests::ReviewRequests + end + + def importer_class + Importer::PullRequests::ReviewRequestImporter + end + + def object_type + :pull_request_review_request + end + end + end + end +end diff --git a/app/workers/gitlab/github_import/stage/import_pull_requests_merged_by_worker.rb b/app/workers/gitlab/github_import/stage/import_pull_requests_merged_by_worker.rb index 8c2d652a689..9b123b5776a 100644 --- a/app/workers/gitlab/github_import/stage/import_pull_requests_merged_by_worker.rb +++ b/app/workers/gitlab/github_import/stage/import_pull_requests_merged_by_worker.rb @@ -24,7 +24,7 @@ module Gitlab AdvanceStageWorker.perform_async( project.id, { waiter.key => waiter.jobs_remaining }, - :pull_request_reviews + :pull_request_review_requests ) end end diff --git a/app/workers/gitlab/github_import/stage/import_pull_requests_review_requests_worker.rb b/app/workers/gitlab/github_import/stage/import_pull_requests_review_requests_worker.rb new file mode 100644 index 00000000000..bcbf5dd471a --- /dev/null +++ b/app/workers/gitlab/github_import/stage/import_pull_requests_review_requests_worker.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Stage + class ImportPullRequestsReviewRequestsWorker # rubocop:disable Scalability/IdempotentWorker + include ApplicationWorker + + data_consistency :always + + sidekiq_options retry: 3 + include GithubImport::Queue + include StageMethods + + # client - An instance of Gitlab::GithubImport::Client. + # project - An instance of Project. + def import(client, project) + waiter = Importer::PullRequests::ReviewRequestsImporter + .new(project, client) + .execute + + project.import_state.refresh_jid_expiration + + AdvanceStageWorker.perform_async( + project.id, + { waiter.key => waiter.jobs_remaining }, + :pull_request_reviews + ) + end + end + end + end +end diff --git a/app/workers/gitlab_performance_bar_stats_worker.rb b/app/workers/gitlab_performance_bar_stats_worker.rb index 6d637ad1586..0b31c159726 100644 --- a/app/workers/gitlab_performance_bar_stats_worker.rb +++ b/app/workers/gitlab_performance_bar_stats_worker.rb @@ -18,7 +18,7 @@ class GitlabPerformanceBarStatsWorker idempotent! def perform(lease_uuid) - Gitlab::Redis::Cache.with do |redis| + with_redis do |redis| request_ids = fetch_request_ids(redis, lease_uuid) stats = Gitlab::PerformanceBar::Stats.new(redis) @@ -30,6 +30,10 @@ class GitlabPerformanceBarStatsWorker private + def with_redis(&block) + Gitlab::Redis::Cache.with(&block) # rubocop:disable CodeReuse/ActiveRecord + end + def fetch_request_ids(redis, lease_uuid) ids = redis.smembers(STATS_KEY) redis.del(STATS_KEY) diff --git a/app/workers/gitlab_shell_worker.rb b/app/workers/gitlab_shell_worker.rb index 1bcaf5a42be..2f396dcdb86 100644 --- a/app/workers/gitlab_shell_worker.rb +++ b/app/workers/gitlab_shell_worker.rb @@ -14,18 +14,12 @@ class GitlabShellWorker # rubocop:disable Scalability/IdempotentWorker loggable_arguments 0 def perform(action, *arg) - # Gitlab::Shell is being removed but we need to continue to process jobs - # enqueued in the previous release, so handle them here. - # - # See https://gitlab.com/gitlab-org/gitlab/-/issues/25095 for more details - if AuthorizedKeysWorker::PERMITTED_ACTIONS.include?(action.to_s) - AuthorizedKeysWorker.new.perform(action, *arg) - - return + if ::Feature.enabled?(:verify_gitlab_shell_worker_method_names) && Gitlab::Shell::PERMITTED_ACTIONS.exclude?(action) + raise(ArgumentError, "#{action} not allowed for #{self.class.name}") end Gitlab::GitalyClient::NamespaceService.allow do - gitlab_shell.__send__(action, *arg) # rubocop:disable GitlabSecurity/PublicSend + gitlab_shell.public_send(action, *arg) # rubocop:disable GitlabSecurity/PublicSend end end end diff --git a/app/workers/incident_management/add_severity_system_note_worker.rb b/app/workers/incident_management/add_severity_system_note_worker.rb index 3a4667bea0a..b2d2d6748ee 100644 --- a/app/workers/incident_management/add_severity_system_note_worker.rb +++ b/app/workers/incident_management/add_severity_system_note_worker.rb @@ -21,7 +21,10 @@ module IncidentManagement user = User.find_by_id(user_id) return unless user - SystemNoteService.change_incident_severity(incident, user) + incident.transaction do + SystemNoteService.change_incident_severity(incident, user) + TimelineEvents::CreateService.change_severity(incident, user) + end end end end diff --git a/app/workers/loose_foreign_keys/cleanup_worker.rb b/app/workers/loose_foreign_keys/cleanup_worker.rb index 0a3a834578a..9a0909598bb 100644 --- a/app/workers/loose_foreign_keys/cleanup_worker.rb +++ b/app/workers/loose_foreign_keys/cleanup_worker.rb @@ -12,7 +12,11 @@ module LooseForeignKeys idempotent! def perform - in_lock(self.class.name.underscore, ttl: ModificationTracker::MAX_RUNTIME, retries: 0) do + # Add small buffer on MAX_RUNTIME to account for single long running + # query or extra worker time after the cleanup. + lock_ttl = ModificationTracker::MAX_RUNTIME + 20.seconds + + in_lock(self.class.name.underscore, ttl: lock_ttl, retries: 0) do stats = {} connection_name, base_model = current_connection_name_and_base_model diff --git a/app/workers/mail_scheduler/notification_service_worker.rb b/app/workers/mail_scheduler/notification_service_worker.rb index 25c9ac5547b..12e8de4491e 100644 --- a/app/workers/mail_scheduler/notification_service_worker.rb +++ b/app/workers/mail_scheduler/notification_service_worker.rb @@ -18,6 +18,12 @@ module MailScheduler def perform(meth, *args) check_arguments!(args) + if ::Feature.enabled?(:verify_mail_scheduler_notification_service_worker_method_names) && + NotificationService.permitted_actions.exclude?(meth.to_sym) + + raise(ArgumentError, "#{meth} not allowed for #{self.class.name}") + end + deserialized_args = ActiveJob::Arguments.deserialize(args) notification_service.public_send(meth, *deserialized_args) # rubocop:disable GitlabSecurity/PublicSend rescue ActiveJob::DeserializationError diff --git a/app/workers/merge_requests/delete_branch_worker.rb b/app/workers/merge_requests/delete_branch_worker.rb new file mode 100644 index 00000000000..6816f9a4b77 --- /dev/null +++ b/app/workers/merge_requests/delete_branch_worker.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +module MergeRequests + class DeleteBranchWorker + include ApplicationWorker + + data_consistency :always + + feature_category :source_code_management + urgency :high + idempotent! + + def perform(merge_request_id, user_id, branch_name, retarget_branch) + merge_request = MergeRequest.find_by_id(merge_request_id) + user = User.find_by_id(user_id) + + return unless merge_request.present? && user.present? + + ::Branches::DeleteService.new(merge_request.source_project, user).execute(branch_name) + + return unless retarget_branch + + ::MergeRequests::RetargetChainService.new(project: merge_request.source_project, current_user: user) + .execute(merge_request) + end + end +end diff --git a/app/workers/merge_requests/delete_source_branch_worker.rb b/app/workers/merge_requests/delete_source_branch_worker.rb index 66392c670b5..96dde413d5b 100644 --- a/app/workers/merge_requests/delete_source_branch_worker.rb +++ b/app/workers/merge_requests/delete_source_branch_worker.rb @@ -18,15 +18,14 @@ class MergeRequests::DeleteSourceBranchWorker # Source branch changed while it's being removed return if merge_request.source_branch_sha != source_branch_sha - delete_service_result = ::Branches::DeleteService.new(merge_request.source_project, user) - .execute(merge_request.source_branch) + if Feature.enabled?(:add_delete_branch_worker, merge_request.source_project) + ::MergeRequests::DeleteBranchWorker.perform_async(merge_request_id, user_id, merge_request.source_branch, true) + else + ::Branches::DeleteService.new(merge_request.source_project, user).execute(merge_request.source_branch) - if Feature.enabled?(:track_delete_source_errors, merge_request.source_project) - delete_service_result.track_exception if delete_service_result&.error? + ::MergeRequests::RetargetChainService.new(project: merge_request.source_project, current_user: user) + .execute(merge_request) end - - ::MergeRequests::RetargetChainService.new(project: merge_request.source_project, current_user: user) - .execute(merge_request) rescue ActiveRecord::RecordNotFound end end diff --git a/app/workers/namespaces/root_statistics_worker.rb b/app/workers/namespaces/root_statistics_worker.rb index e1271dae335..e3aa8a1f779 100644 --- a/app/workers/namespaces/root_statistics_worker.rb +++ b/app/workers/namespaces/root_statistics_worker.rb @@ -4,13 +4,14 @@ module Namespaces class RootStatisticsWorker include ApplicationWorker - data_consistency :always + data_consistency :sticky, feature_flag: :root_statistics_worker_read_replica sidekiq_options retry: 3 queue_namespace :update_namespace_statistics feature_category :source_code_management idempotent! + deduplicate :until_executed, if_deduplicated: :reschedule_once def perform(namespace_id) namespace = Namespace.find(namespace_id) diff --git a/app/workers/onboarding/issue_created_worker.rb b/app/workers/onboarding/issue_created_worker.rb index ff39fefad81..73e96850786 100644 --- a/app/workers/onboarding/issue_created_worker.rb +++ b/app/workers/onboarding/issue_created_worker.rb @@ -22,6 +22,3 @@ module Onboarding end end end - -# remove in %15.6 as per https://gitlab.com/gitlab-org/gitlab/-/issues/372432 -Namespaces::OnboardingIssueCreatedWorker = Onboarding::IssueCreatedWorker diff --git a/app/workers/onboarding/pipeline_created_worker.rb b/app/workers/onboarding/pipeline_created_worker.rb index 6bd5863b0e0..c6e84882d6f 100644 --- a/app/workers/onboarding/pipeline_created_worker.rb +++ b/app/workers/onboarding/pipeline_created_worker.rb @@ -22,6 +22,3 @@ module Onboarding end end end - -# remove in %15.6 as per https://gitlab.com/gitlab-org/gitlab/-/issues/372432 -Namespaces::OnboardingPipelineCreatedWorker = Onboarding::PipelineCreatedWorker diff --git a/app/workers/onboarding/progress_worker.rb b/app/workers/onboarding/progress_worker.rb index 525934c4a7c..34503bfa451 100644 --- a/app/workers/onboarding/progress_worker.rb +++ b/app/workers/onboarding/progress_worker.rb @@ -23,6 +23,3 @@ module Onboarding end end end - -# remove in %15.6 as per https://gitlab.com/gitlab-org/gitlab/-/issues/372432 -Namespaces::OnboardingProgressWorker = Onboarding::ProgressWorker diff --git a/app/workers/onboarding/user_added_worker.rb b/app/workers/onboarding/user_added_worker.rb index 38e9cd063ea..b096bf752dc 100644 --- a/app/workers/onboarding/user_added_worker.rb +++ b/app/workers/onboarding/user_added_worker.rb @@ -19,6 +19,3 @@ module Onboarding end end end - -# remove in %15.6 as per https://gitlab.com/gitlab-org/gitlab/-/issues/372432 -Namespaces::OnboardingUserAddedWorker = Onboarding::UserAddedWorker diff --git a/app/workers/pages_worker.rb b/app/workers/pages_worker.rb index 3aff4b42629..adb6d38fd12 100644 --- a/app/workers/pages_worker.rb +++ b/app/workers/pages_worker.rb @@ -11,7 +11,7 @@ class PagesWorker # rubocop:disable Scalability/IdempotentWorker worker_resource_boundary :cpu def perform(action, *arg) - send(action, *arg) # rubocop:disable GitlabSecurity/PublicSend + deploy(*arg) if action == 'deploy' end def deploy(build_id) diff --git a/app/workers/projects/inactive_projects_deletion_cron_worker.rb b/app/workers/projects/inactive_projects_deletion_cron_worker.rb index ba6d44ec4a5..af62efeb089 100644 --- a/app/workers/projects/inactive_projects_deletion_cron_worker.rb +++ b/app/workers/projects/inactive_projects_deletion_cron_worker.rb @@ -90,22 +90,26 @@ module Projects end def save_last_processed_project_id(project_id) - Gitlab::Redis::Cache.with do |redis| + with_redis do |redis| redis.set(LAST_PROCESSED_INACTIVE_PROJECT_REDIS_KEY, project_id) end end def last_processed_project_id - Gitlab::Redis::Cache.with do |redis| + with_redis do |redis| redis.get(LAST_PROCESSED_INACTIVE_PROJECT_REDIS_KEY).to_i end end def reset_last_processed_project_id - Gitlab::Redis::Cache.with do |redis| + with_redis do |redis| redis.del(LAST_PROCESSED_INACTIVE_PROJECT_REDIS_KEY) end end + + def with_redis(&block) + Gitlab::Redis::Cache.with(&block) # rubocop:disable CodeReuse/ActiveRecord + end end end diff --git a/app/workers/projects/post_creation_worker.rb b/app/workers/projects/post_creation_worker.rb index 3a39bd17ce3..886919ecace 100644 --- a/app/workers/projects/post_creation_worker.rb +++ b/app/workers/projects/post_creation_worker.rb @@ -17,6 +17,7 @@ module Projects return unless project create_prometheus_integration(project) + create_incident_management_timeline_event_tags(project) end private @@ -34,5 +35,19 @@ module Projects rescue ActiveRecord::RecordInvalid => e Gitlab::ErrorTracking.track_exception(e, extra: { project_id: project.id }) end + + def create_incident_management_timeline_event_tags(project) + tags = project.incident_management_timeline_event_tags.pluck_names + start_time_name = ::IncidentManagement::TimelineEventTag::START_TIME_TAG_NAME + end_time_name = ::IncidentManagement::TimelineEventTag::END_TIME_TAG_NAME + + project.incident_management_timeline_event_tags.new(name: start_time_name) unless tags.include?(start_time_name) + + project.incident_management_timeline_event_tags.new(name: end_time_name) unless tags.include?(end_time_name) + + project.save! + rescue StandardError => e + Gitlab::ErrorTracking.track_exception(e, extra: { project_id: project.id }) + end end end diff --git a/app/workers/repository_import_worker.rb b/app/workers/repository_import_worker.rb index 413bb135943..5e89b9f3362 100644 --- a/app/workers/repository_import_worker.rb +++ b/app/workers/repository_import_worker.rb @@ -33,11 +33,9 @@ class RepositoryImportWorker # rubocop:disable Scalability/IdempotentWorker if result[:status] == :error fail_import(result[:message]) - - raise result[:message] + else + project.after_import end - - project.after_import end private diff --git a/app/workers/run_pipeline_schedule_worker.rb b/app/workers/run_pipeline_schedule_worker.rb index 35e3e633c70..8974ddce47b 100644 --- a/app/workers/run_pipeline_schedule_worker.rb +++ b/app/workers/run_pipeline_schedule_worker.rb @@ -21,13 +21,14 @@ class RunPipelineScheduleWorker # rubocop:disable Scalability/IdempotentWorker end def run_pipeline_schedule(schedule, user) - Ci::CreatePipelineService.new(schedule.project, - user, - ref: schedule.ref) - .execute!(:schedule, ignore_skip_ci: true, save_on_errors: false, schedule: schedule) - rescue Ci::CreatePipelineService::CreateError => e + response = Ci::CreatePipelineService + .new(schedule.project, user, ref: schedule.ref) + .execute(:schedule, ignore_skip_ci: true, save_on_errors: false, schedule: schedule) + + return response if response.payload.persisted? + # This is a user operation error such as corrupted .gitlab-ci.yml. Log the error for debugging purpose. - log_extra_metadata_on_done(:pipeline_creation_error, e) + log_extra_metadata_on_done(:pipeline_creation_error, response.message) rescue StandardError => e error(schedule, e) end diff --git a/app/workers/update_highest_role_worker.rb b/app/workers/update_highest_role_worker.rb index 064b8203d4d..a05c9c7a1e7 100644 --- a/app/workers/update_highest_role_worker.rb +++ b/app/workers/update_highest_role_worker.rb @@ -7,7 +7,7 @@ class UpdateHighestRoleWorker sidekiq_options retry: 3 - feature_category :utilization + feature_category :subscription_usage_reports urgency :high weight 2 diff --git a/app/workers/users/deactivate_dormant_users_worker.rb b/app/workers/users/deactivate_dormant_users_worker.rb index b14b7e67450..c3799480b12 100644 --- a/app/workers/users/deactivate_dormant_users_worker.rb +++ b/app/workers/users/deactivate_dormant_users_worker.rb @@ -8,7 +8,7 @@ module Users include CronjobQueue - feature_category :utilization + feature_category :subscription_cost_management def perform return if Gitlab.com? diff --git a/app/workers/users/migrate_records_to_ghost_user_in_batches_worker.rb b/app/workers/users/migrate_records_to_ghost_user_in_batches_worker.rb index ddddfc106ae..d9a80b6e899 100644 --- a/app/workers/users/migrate_records_to_ghost_user_in_batches_worker.rb +++ b/app/workers/users/migrate_records_to_ghost_user_in_batches_worker.rb @@ -12,8 +12,6 @@ module Users idempotent! def perform - return unless Feature.enabled?(:user_destroy_with_limited_execution_time_worker) - in_lock(self.class.name.underscore, ttl: Gitlab::Utils::ExecutionTracker::MAX_RUNTIME, retries: 0) do Users::MigrateRecordsToGhostUserInBatchesService.new.execute end |