diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2020-10-21 10:08:36 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2020-10-21 10:08:36 +0300 |
commit | 48aff82709769b098321c738f3444b9bdaa694c6 (patch) | |
tree | e00c7c43e2d9b603a5a6af576b1685e400410dee /app/workers | |
parent | 879f5329ee916a948223f8f43d77fba4da6cd028 (diff) |
Add latest changes from gitlab-org/gitlab@13-5-stable-eev13.5.0-rc42
Diffstat (limited to 'app/workers')
54 files changed, 892 insertions, 54 deletions
diff --git a/app/workers/all_queues.yml b/app/workers/all_queues.yml index 11bf797fb90..30b89f37562 100644 --- a/app/workers/all_queues.yml +++ b/app/workers/all_queues.yml @@ -147,6 +147,14 @@ :weight: 1 :idempotent: :tags: [] +- :name: cronjob:ci_schedule_delete_objects_cron + :feature_category: :continuous_integration + :has_external_dependencies: + :urgency: :low + :resource_boundary: :unknown + :weight: 1 + :idempotent: true + :tags: [] - :name: cronjob:container_expiration_policy :feature_category: :container_registry :has_external_dependencies: @@ -211,6 +219,14 @@ :weight: 1 :idempotent: :tags: [] +- :name: cronjob:member_invitation_reminder_emails + :feature_category: :subgroups + :has_external_dependencies: + :urgency: :low + :resource_boundary: :unknown + :weight: 1 + :idempotent: + :tags: [] - :name: cronjob:metrics_dashboard_schedule_annotations_prune :feature_category: :metrics :has_external_dependencies: @@ -419,6 +435,22 @@ :weight: 1 :idempotent: true :tags: [] +- :name: deployment:deployments_drop_older_deployments + :feature_category: :continuous_delivery + :has_external_dependencies: + :urgency: :low + :resource_boundary: :unknown + :weight: 3 + :idempotent: + :tags: [] +- :name: deployment:deployments_execute_hooks + :feature_category: :continuous_delivery + :has_external_dependencies: + :urgency: :low + :resource_boundary: :cpu + :weight: 3 + :idempotent: + :tags: [] - :name: deployment:deployments_finished :feature_category: :continuous_delivery :has_external_dependencies: @@ -435,6 +467,14 @@ :weight: 3 :idempotent: :tags: [] +- :name: deployment:deployments_link_merge_request + :feature_category: :continuous_delivery + :has_external_dependencies: + :urgency: :low + :resource_boundary: :cpu + :weight: 3 + :idempotent: true + :tags: [] - :name: deployment:deployments_success :feature_category: :continuous_delivery :has_external_dependencies: @@ -443,6 +483,14 @@ :weight: 3 :idempotent: :tags: [] +- :name: deployment:deployments_update_environment + :feature_category: :continuous_delivery + :has_external_dependencies: + :urgency: :low + :resource_boundary: :cpu + :weight: 3 + :idempotent: true + :tags: [] - :name: gcp_cluster:cluster_configure_istio :feature_category: :kubernetes_management :has_external_dependencies: true @@ -723,6 +771,14 @@ :weight: 2 :idempotent: true :tags: [] +- :name: incident_management:incident_management_add_severity_system_note + :feature_category: :incident_management + :has_external_dependencies: + :urgency: :low + :resource_boundary: :unknown + :weight: 2 + :idempotent: + :tags: [] - :name: incident_management:incident_management_pager_duty_process_incident :feature_category: :incident_management :has_external_dependencies: @@ -906,7 +962,8 @@ :resource_boundary: :unknown :weight: 1 :idempotent: - :tags: [] + :tags: + - :requires_disk_io - :name: pipeline_background:ci_build_report_result :feature_category: :continuous_integration :has_external_dependencies: @@ -914,7 +971,8 @@ :resource_boundary: :unknown :weight: 1 :idempotent: true - :tags: [] + :tags: + - :requires_disk_io - :name: pipeline_background:ci_build_trace_chunk_flush :feature_category: :continuous_integration :has_external_dependencies: @@ -994,7 +1052,8 @@ :resource_boundary: :unknown :weight: 3 :idempotent: - :tags: [] + :tags: + - :requires_disk_io - :name: pipeline_default:build_trace_sections :feature_category: :continuous_integration :has_external_dependencies: @@ -1002,7 +1061,8 @@ :resource_boundary: :unknown :weight: 3 :idempotent: - :tags: [] + :tags: + - :requires_disk_io - :name: pipeline_default:ci_create_cross_project_pipeline :feature_category: :continuous_integration :has_external_dependencies: @@ -1259,6 +1319,15 @@ :resource_boundary: :unknown :weight: 2 :idempotent: + :tags: + - :requires_disk_io +- :name: ci_delete_objects + :feature_category: :continuous_integration + :has_external_dependencies: + :urgency: :low + :resource_boundary: :unknown + :weight: 1 + :idempotent: true :tags: [] - :name: create_commit_signature :feature_category: :source_code_management @@ -1324,6 +1393,14 @@ :weight: 1 :idempotent: :tags: [] +- :name: design_management_copy_design_collection + :feature_category: :design_management + :has_external_dependencies: + :urgency: :low + :resource_boundary: :unknown + :weight: 1 + :idempotent: true + :tags: [] - :name: design_management_new_version :feature_category: :design_management :has_external_dependencies: @@ -1340,6 +1417,22 @@ :weight: 1 :idempotent: :tags: [] +- :name: disallow_two_factor_for_group + :feature_category: :subgroups + :has_external_dependencies: + :urgency: :low + :resource_boundary: :unknown + :weight: 1 + :idempotent: true + :tags: [] +- :name: disallow_two_factor_for_subgroups + :feature_category: :subgroups + :has_external_dependencies: + :urgency: :low + :resource_boundary: :unknown + :weight: 1 + :idempotent: true + :tags: [] - :name: email_receiver :feature_category: :issue_tracking :has_external_dependencies: @@ -1371,7 +1464,8 @@ :resource_boundary: :unknown :weight: 1 :idempotent: - :tags: [] + :tags: + - :requires_disk_io - :name: export_csv :feature_category: :issue_tracking :has_external_dependencies: @@ -1435,7 +1529,8 @@ :resource_boundary: :unknown :weight: 1 :idempotent: - :tags: [] + :tags: + - :requires_disk_io - :name: group_export :feature_category: :importers :has_external_dependencies: @@ -1476,6 +1571,14 @@ :weight: 1 :idempotent: :tags: [] +- :name: issuable_export_csv + :feature_category: :issue_tracking + :has_external_dependencies: + :urgency: :low + :resource_boundary: :cpu + :weight: 1 + :idempotent: + :tags: [] - :name: issue_placement :feature_category: :issue_tracking :has_external_dependencies: @@ -1532,6 +1635,14 @@ :weight: 1 :idempotent: true :tags: [] +- :name: metrics_dashboard_sync_dashboards + :feature_category: :metrics + :has_external_dependencies: + :urgency: :low + :resource_boundary: :unknown + :weight: 1 + :idempotent: true + :tags: [] - :name: migrate_external_diffs :feature_category: :source_code_management :has_external_dependencies: @@ -1579,7 +1690,8 @@ :resource_boundary: :unknown :weight: 1 :idempotent: - :tags: [] + :tags: + - :requires_disk_io - :name: pages_domain_ssl_renewal :feature_category: :pages :has_external_dependencies: @@ -1587,7 +1699,8 @@ :resource_boundary: :unknown :weight: 1 :idempotent: - :tags: [] + :tags: + - :requires_disk_io - :name: pages_domain_verification :feature_category: :pages :has_external_dependencies: @@ -1595,7 +1708,8 @@ :resource_boundary: :unknown :weight: 1 :idempotent: - :tags: [] + :tags: + - :requires_disk_io - :name: pages_remove :feature_category: :pages :has_external_dependencies: @@ -1667,7 +1781,8 @@ :resource_boundary: :unknown :weight: 1 :idempotent: - :tags: [] + :tags: + - :requires_disk_io - :name: project_export :feature_category: :importers :has_external_dependencies: @@ -1708,6 +1823,30 @@ :weight: 1 :idempotent: true :tags: [] +- :name: propagate_integration_group + :feature_category: :integrations + :has_external_dependencies: + :urgency: :low + :resource_boundary: :unknown + :weight: 1 + :idempotent: true + :tags: [] +- :name: propagate_integration_inherit + :feature_category: :integrations + :has_external_dependencies: + :urgency: :low + :resource_boundary: :unknown + :weight: 1 + :idempotent: true + :tags: [] +- :name: propagate_integration_project + :feature_category: :integrations + :has_external_dependencies: + :urgency: :low + :resource_boundary: :unknown + :weight: 1 + :idempotent: true + :tags: [] - :name: propagate_service_template :feature_category: :integrations :has_external_dependencies: @@ -1860,6 +1999,14 @@ :weight: 1 :idempotent: :tags: [] +- :name: web_hooks_destroy + :feature_category: :integrations + :has_external_dependencies: + :urgency: :low + :resource_boundary: :unknown + :weight: 1 + :idempotent: true + :tags: [] - :name: x509_certificate_revoke :feature_category: :source_code_management :has_external_dependencies: diff --git a/app/workers/analytics/instance_statistics/count_job_trigger_worker.rb b/app/workers/analytics/instance_statistics/count_job_trigger_worker.rb index a9976c6e5cb..bf57619fc6e 100644 --- a/app/workers/analytics/instance_statistics/count_job_trigger_worker.rb +++ b/app/workers/analytics/instance_statistics/count_job_trigger_worker.rb @@ -14,13 +14,10 @@ module Analytics idempotent! def perform - return if Feature.disabled?(:store_instance_statistics_measurements, default_enabled: true) - recorded_at = Time.zone.now - measurement_identifiers = Analytics::InstanceStatistics::Measurement.identifiers worker_arguments = Gitlab::Analytics::InstanceStatistics::WorkersArgumentBuilder.new( - measurement_identifiers: measurement_identifiers.values, + measurement_identifiers: ::Analytics::InstanceStatistics::Measurement.measurement_identifier_values, recorded_at: recorded_at ).execute diff --git a/app/workers/archive_trace_worker.rb b/app/workers/archive_trace_worker.rb index 3ddb5686bf2..b0c5bef336a 100644 --- a/app/workers/archive_trace_worker.rb +++ b/app/workers/archive_trace_worker.rb @@ -4,6 +4,8 @@ class ArchiveTraceWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker include PipelineBackgroundQueue + tags :requires_disk_io + # rubocop: disable CodeReuse/ActiveRecord def perform(job_id) Ci::Build.without_archived_trace.find_by(id: job_id).try do |job| diff --git a/app/workers/authorized_project_update/periodic_recalculate_worker.rb b/app/workers/authorized_project_update/periodic_recalculate_worker.rb index 0d1ad67d7bb..78ffdbca4d6 100644 --- a/app/workers/authorized_project_update/periodic_recalculate_worker.rb +++ b/app/workers/authorized_project_update/periodic_recalculate_worker.rb @@ -12,9 +12,7 @@ module AuthorizedProjectUpdate idempotent! def perform - if ::Feature.enabled?(:periodic_project_authorization_recalculation, default_enabled: true) - AuthorizedProjectUpdate::PeriodicRecalculateService.new.execute - end + AuthorizedProjectUpdate::PeriodicRecalculateService.new.execute end end end diff --git a/app/workers/authorized_project_update/user_refresh_over_user_range_worker.rb b/app/workers/authorized_project_update/user_refresh_over_user_range_worker.rb index 336b1c5443e..9bd1ad2ed30 100644 --- a/app/workers/authorized_project_update/user_refresh_over_user_range_worker.rb +++ b/app/workers/authorized_project_update/user_refresh_over_user_range_worker.rb @@ -12,9 +12,7 @@ module AuthorizedProjectUpdate idempotent! def perform(start_user_id, end_user_id) - if ::Feature.enabled?(:periodic_project_authorization_recalculation, default_enabled: true) - AuthorizedProjectUpdate::RecalculateForUserRangeService.new(start_user_id, end_user_id).execute - end + AuthorizedProjectUpdate::RecalculateForUserRangeService.new(start_user_id, end_user_id).execute end end end diff --git a/app/workers/background_migration_worker.rb b/app/workers/background_migration_worker.rb index bff864ba420..74a12dbff77 100644 --- a/app/workers/background_migration_worker.rb +++ b/app/workers/background_migration_worker.rb @@ -23,7 +23,9 @@ class BackgroundMigrationWorker # rubocop:disable Scalability/IdempotentWorker # # class_name - The class name of the background migration to run. # arguments - The arguments to pass to the migration class. - def perform(class_name, arguments = []) + # lease_attempts - The number of times we will try to obtain an exclusive + # lease on the class before running anyway. Pass 0 to always run. + def perform(class_name, arguments = [], lease_attempts = 5) with_context(caller_id: class_name.to_s) do should_perform, ttl = perform_and_ttl(class_name) diff --git a/app/workers/build_coverage_worker.rb b/app/workers/build_coverage_worker.rb index 7d893024abc..d63d8549f09 100644 --- a/app/workers/build_coverage_worker.rb +++ b/app/workers/build_coverage_worker.rb @@ -4,6 +4,8 @@ class BuildCoverageWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker include PipelineQueue + tags :requires_disk_io + # rubocop: disable CodeReuse/ActiveRecord def perform(build_id) Ci::Build.find_by(id: build_id)&.update_coverage diff --git a/app/workers/build_finished_worker.rb b/app/workers/build_finished_worker.rb index d0f7d65aed6..d7a5fcf4f18 100644 --- a/app/workers/build_finished_worker.rb +++ b/app/workers/build_finished_worker.rb @@ -9,6 +9,8 @@ class BuildFinishedWorker # rubocop:disable Scalability/IdempotentWorker worker_resource_boundary :cpu tags :requires_disk_io + ARCHIVE_TRACES_IN = 2.minutes.freeze + # rubocop: disable CodeReuse/ActiveRecord def perform(build_id) Ci::Build.find_by(id: build_id).try do |build| @@ -33,9 +35,22 @@ class BuildFinishedWorker # rubocop:disable Scalability/IdempotentWorker # We execute these async as these are independent operations. BuildHooksWorker.perform_async(build.id) - ArchiveTraceWorker.perform_async(build.id) ExpirePipelineCacheWorker.perform_async(build.pipeline_id) if build.pipeline.cacheable? ChatNotificationWorker.perform_async(build.id) if build.pipeline.chat? + + ## + # We want to delay sending a build trace to object storage operation to + # validate that this fixes a race condition between this and flushing live + # trace chunks and chunks being removed after consolidation and putting + # them into object storage archive. + # + # TODO This is temporary fix we should improve later, after we validate + # that this is indeed the culprit. + # + # See https://gitlab.com/gitlab-org/gitlab/-/issues/267112 for more + # details. + # + ArchiveTraceWorker.perform_in(ARCHIVE_TRACES_IN, build.id) end end diff --git a/app/workers/build_trace_sections_worker.rb b/app/workers/build_trace_sections_worker.rb index c25f77974e9..59f019b827e 100644 --- a/app/workers/build_trace_sections_worker.rb +++ b/app/workers/build_trace_sections_worker.rb @@ -4,6 +4,8 @@ class BuildTraceSectionsWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker include PipelineQueue + tags :requires_disk_io + # rubocop: disable CodeReuse/ActiveRecord def perform(build_id) Ci::Build.find_by(id: build_id)&.parse_trace_sections! diff --git a/app/workers/chat_notification_worker.rb b/app/workers/chat_notification_worker.rb index 5fab437f49f..94a0197b862 100644 --- a/app/workers/chat_notification_worker.rb +++ b/app/workers/chat_notification_worker.rb @@ -7,6 +7,7 @@ class ChatNotificationWorker # rubocop:disable Scalability/IdempotentWorker sidekiq_options retry: false feature_category :chatops + tags :requires_disk_io urgency :low # Can't be high as it has external dependencies weight 2 worker_has_external_dependencies! diff --git a/app/workers/ci/build_report_result_worker.rb b/app/workers/ci/build_report_result_worker.rb index 60387936d0b..01a45490541 100644 --- a/app/workers/ci/build_report_result_worker.rb +++ b/app/workers/ci/build_report_result_worker.rb @@ -5,6 +5,8 @@ module Ci include ApplicationWorker include PipelineBackgroundQueue + tags :requires_disk_io + idempotent! def perform(build_id) diff --git a/app/workers/ci/build_trace_chunk_flush_worker.rb b/app/workers/ci/build_trace_chunk_flush_worker.rb index 2908c7c2d0b..89400247a7b 100644 --- a/app/workers/ci/build_trace_chunk_flush_worker.rb +++ b/app/workers/ci/build_trace_chunk_flush_worker.rb @@ -8,8 +8,8 @@ module Ci idempotent! # rubocop: disable CodeReuse/ActiveRecord - def perform(chunk_id) - ::Ci::BuildTraceChunk.find_by(id: chunk_id).try do |chunk| + def perform(id) + ::Ci::BuildTraceChunk.find_by(id: id).try do |chunk| chunk.persist_data! end end diff --git a/app/workers/ci/delete_objects_worker.rb b/app/workers/ci/delete_objects_worker.rb new file mode 100644 index 00000000000..e34be33b438 --- /dev/null +++ b/app/workers/ci/delete_objects_worker.rb @@ -0,0 +1,38 @@ +# frozen_string_literal: true + +module Ci + class DeleteObjectsWorker + include ApplicationWorker + include LimitedCapacity::Worker + + feature_category :continuous_integration + idempotent! + + def perform_work(*args) + service.execute + end + + def remaining_work_count(*args) + @remaining_work_count ||= service + .remaining_batches_count(max_batch_count: remaining_capacity) + end + + def max_running_jobs + if ::Feature.enabled?(:ci_delete_objects_low_concurrency) + 2 + elsif ::Feature.enabled?(:ci_delete_objects_medium_concurrency) + 20 + elsif ::Feature.enabled?(:ci_delete_objects_high_concurrency) + 50 + else + 0 + end + end + + private + + def service + @service ||= DeleteObjectsService.new + end + end +end diff --git a/app/workers/ci/schedule_delete_objects_cron_worker.rb b/app/workers/ci/schedule_delete_objects_cron_worker.rb new file mode 100644 index 00000000000..fa0b15deb56 --- /dev/null +++ b/app/workers/ci/schedule_delete_objects_cron_worker.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +module Ci + class ScheduleDeleteObjectsCronWorker + include ApplicationWorker + # rubocop:disable Scalability/CronWorkerContext + # This worker does not perform work scoped to a context + include CronjobQueue + # rubocop:enable Scalability/CronWorkerContext + + feature_category :continuous_integration + idempotent! + + def perform(*args) + Ci::DeleteObjectsWorker.perform_with_capacity(*args) + end + end +end diff --git a/app/workers/cleanup_container_repository_worker.rb b/app/workers/cleanup_container_repository_worker.rb index 4469ea8cff9..80cc296fff5 100644 --- a/app/workers/cleanup_container_repository_worker.rb +++ b/app/workers/cleanup_container_repository_worker.rb @@ -16,9 +16,17 @@ class CleanupContainerRepositoryWorker # rubocop:disable Scalability/IdempotentW return unless valid? - Projects::ContainerRepository::CleanupTagsService + if run_by_container_expiration_policy? + container_repository.start_expiration_policy! + end + + result = Projects::ContainerRepository::CleanupTagsService .new(project, current_user, params) .execute(container_repository) + + if run_by_container_expiration_policy? && result[:status] == :success + container_repository.reset_expiration_policy_started_at! + end end private @@ -30,7 +38,7 @@ class CleanupContainerRepositoryWorker # rubocop:disable Scalability/IdempotentW end def run_by_container_expiration_policy? - @params['container_expiration_policy'] && container_repository && project + @params['container_expiration_policy'] && container_repository.present? && project.present? end def project diff --git a/app/workers/concerns/limited_capacity/job_tracker.rb b/app/workers/concerns/limited_capacity/job_tracker.rb new file mode 100644 index 00000000000..96b6e1a2024 --- /dev/null +++ b/app/workers/concerns/limited_capacity/job_tracker.rb @@ -0,0 +1,74 @@ +# frozen_string_literal: true +module LimitedCapacity + class JobTracker # rubocop:disable Scalability/IdempotentWorker + include Gitlab::Utils::StrongMemoize + + def initialize(namespace) + @namespace = namespace + end + + def register(jid) + _added, @count = with_redis_pipeline do |redis| + register_job_keys(redis, jid) + get_job_count(redis) + end + end + + def remove(jid) + _removed, @count = with_redis_pipeline do |redis| + remove_job_keys(redis, jid) + get_job_count(redis) + end + end + + def clean_up + completed_jids = Gitlab::SidekiqStatus.completed_jids(running_jids) + return unless completed_jids.any? + + _removed, @count = with_redis_pipeline do |redis| + remove_job_keys(redis, completed_jids) + get_job_count(redis) + end + end + + def count + @count ||= with_redis { |redis| get_job_count(redis) } + end + + def running_jids + with_redis do |redis| + redis.smembers(counter_key) + end + end + + private + + attr_reader :namespace + + def counter_key + "worker:#{namespace.to_s.underscore}:running" + end + + def get_job_count(redis) + redis.scard(counter_key) + end + + def register_job_keys(redis, keys) + redis.sadd(counter_key, keys) + end + + def remove_job_keys(redis, keys) + redis.srem(counter_key, keys) + end + + def with_redis(&block) + Gitlab::Redis::Queues.with(&block) # rubocop: disable CodeReuse/ActiveRecord + end + + def with_redis_pipeline(&block) + with_redis do |redis| + redis.pipelined(&block) + end + end + end +end diff --git a/app/workers/concerns/limited_capacity/worker.rb b/app/workers/concerns/limited_capacity/worker.rb new file mode 100644 index 00000000000..c0d6bfff2f5 --- /dev/null +++ b/app/workers/concerns/limited_capacity/worker.rb @@ -0,0 +1,164 @@ +# frozen_string_literal: true + +# Usage: +# +# Worker that performs the tasks: +# +# class DummyWorker +# include ApplicationWorker +# include LimitedCapacity::Worker +# +# # For each job that raises any error, a worker instance will be disabled +# # until the next schedule-run. +# # If you wish to get around this, exceptions must by handled by the implementer. +# # +# def perform_work(*args) +# end +# +# def remaining_work_count(*args) +# 5 +# end +# +# def max_running_jobs +# 25 +# end +# end +# +# Cron worker to fill the pool of regular workers: +# +# class ScheduleDummyCronWorker +# include ApplicationWorker +# include CronjobQueue +# +# def perform(*args) +# DummyWorker.perform_with_capacity(*args) +# end +# end +# + +module LimitedCapacity + module Worker + extend ActiveSupport::Concern + include Gitlab::Utils::StrongMemoize + + included do + # Disable Sidekiq retries, log the error, and send the job to the dead queue. + # This is done to have only one source that produces jobs and because the slot + # would be occupied by a job that will be performed in the distant future. + # We let the cron worker enqueue new jobs, this could be seen as our retry and + # back off mechanism because the job might fail again if executed immediately. + sidekiq_options retry: 0 + deduplicate :none + end + + class_methods do + def perform_with_capacity(*args) + worker = self.new + worker.remove_failed_jobs + worker.report_prometheus_metrics(*args) + required_jobs_count = worker.required_jobs_count(*args) + + arguments = Array.new(required_jobs_count) { args } + self.bulk_perform_async(arguments) # rubocop:disable Scalability/BulkPerformWithContext + end + end + + def perform(*args) + return unless has_capacity? + + job_tracker.register(jid) + perform_work(*args) + rescue => exception + raise + ensure + job_tracker.remove(jid) + report_prometheus_metrics + re_enqueue(*args) unless exception + end + + def perform_work(*args) + raise NotImplementedError + end + + def remaining_work_count(*args) + raise NotImplementedError + end + + def max_running_jobs + raise NotImplementedError + end + + def has_capacity? + remaining_capacity > 0 + end + + def remaining_capacity + [ + max_running_jobs - running_jobs_count - self.class.queue_size, + 0 + ].max + end + + def has_work?(*args) + remaining_work_count(*args) > 0 + end + + def remove_failed_jobs + job_tracker.clean_up + end + + def report_prometheus_metrics(*args) + running_jobs_gauge.set(prometheus_labels, running_jobs_count) + remaining_work_gauge.set(prometheus_labels, remaining_work_count(*args)) + max_running_jobs_gauge.set(prometheus_labels, max_running_jobs) + end + + def required_jobs_count(*args) + [ + remaining_work_count(*args), + remaining_capacity + ].min + end + + private + + def running_jobs_count + job_tracker.count + end + + def job_tracker + strong_memoize(:job_tracker) do + JobTracker.new(self.class.name) + end + end + + def re_enqueue(*args) + return unless has_capacity? + return unless has_work?(*args) + + self.class.perform_async(*args) + end + + def running_jobs_gauge + strong_memoize(:running_jobs_gauge) do + Gitlab::Metrics.gauge(:limited_capacity_worker_running_jobs, 'Number of running jobs') + end + end + + def max_running_jobs_gauge + strong_memoize(:max_running_jobs_gauge) do + Gitlab::Metrics.gauge(:limited_capacity_worker_max_running_jobs, 'Maximum number of running jobs') + end + end + + def remaining_work_gauge + strong_memoize(:remaining_work_gauge) do + Gitlab::Metrics.gauge(:limited_capacity_worker_remaining_work_count, 'Number of jobs waiting to be enqueued') + end + end + + def prometheus_labels + { worker: self.class.name } + end + end +end diff --git a/app/workers/container_expiration_policy_worker.rb b/app/workers/container_expiration_policy_worker.rb index 96590e165ae..61ba27f00d2 100644 --- a/app/workers/container_expiration_policy_worker.rb +++ b/app/workers/container_expiration_policy_worker.rb @@ -7,13 +7,15 @@ class ContainerExpirationPolicyWorker # rubocop:disable Scalability/IdempotentWo feature_category :container_registry def perform - ContainerExpirationPolicy.runnable_schedules.preloaded.find_each do |container_expiration_policy| - with_context(project: container_expiration_policy.project, - user: container_expiration_policy.project.owner) do |project:, user:| - ContainerExpirationPolicyService.new(project, user) - .execute(container_expiration_policy) - rescue ContainerExpirationPolicyService::InvalidPolicyError => e - Gitlab::ErrorTracking.log_exception(e, container_expiration_policy_id: container_expiration_policy.id) + ContainerExpirationPolicy.executable.preloaded.each_batch do |relation| + relation.each do |container_expiration_policy| + with_context(project: container_expiration_policy.project, + user: container_expiration_policy.project.owner) do |project:, user:| + ContainerExpirationPolicyService.new(project, user) + .execute(container_expiration_policy) + rescue ContainerExpirationPolicyService::InvalidPolicyError => e + Gitlab::ErrorTracking.log_exception(e, container_expiration_policy_id: container_expiration_policy.id) + end end end end diff --git a/app/workers/deployments/drop_older_deployments_worker.rb b/app/workers/deployments/drop_older_deployments_worker.rb new file mode 100644 index 00000000000..d6cd92c1da4 --- /dev/null +++ b/app/workers/deployments/drop_older_deployments_worker.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +module Deployments + class DropOlderDeploymentsWorker # rubocop:disable Scalability/IdempotentWorker + include ApplicationWorker + + queue_namespace :deployment + feature_category :continuous_delivery + + def perform(deployment_id) + Deployments::OlderDeploymentsDropService.new(deployment_id).execute + end + end +end diff --git a/app/workers/deployments/execute_hooks_worker.rb b/app/workers/deployments/execute_hooks_worker.rb new file mode 100644 index 00000000000..6be05232321 --- /dev/null +++ b/app/workers/deployments/execute_hooks_worker.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module Deployments + class ExecuteHooksWorker # rubocop:disable Scalability/IdempotentWorker + include ApplicationWorker + + queue_namespace :deployment + feature_category :continuous_delivery + worker_resource_boundary :cpu + + def perform(deployment_id) + if (deploy = Deployment.find_by_id(deployment_id)) + deploy.execute_hooks + end + end + end +end diff --git a/app/workers/deployments/finished_worker.rb b/app/workers/deployments/finished_worker.rb index 0be420af718..62c886010a3 100644 --- a/app/workers/deployments/finished_worker.rb +++ b/app/workers/deployments/finished_worker.rb @@ -1,5 +1,7 @@ # frozen_string_literal: true +# This worker is deprecated and will be removed in 14.0 +# See: https://gitlab.com/gitlab-org/gitlab/-/issues/266381 module Deployments class FinishedWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker diff --git a/app/workers/deployments/forward_deployment_worker.rb b/app/workers/deployments/forward_deployment_worker.rb index a6f246dbbbd..dd01fcbbafe 100644 --- a/app/workers/deployments/forward_deployment_worker.rb +++ b/app/workers/deployments/forward_deployment_worker.rb @@ -1,5 +1,7 @@ # frozen_string_literal: true +# This worker is deprecated and will be removed in 14.0 +# See: https://gitlab.com/gitlab-org/gitlab/-/issues/266381 module Deployments class ForwardDeploymentWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker diff --git a/app/workers/deployments/link_merge_request_worker.rb b/app/workers/deployments/link_merge_request_worker.rb new file mode 100644 index 00000000000..4723691a0bb --- /dev/null +++ b/app/workers/deployments/link_merge_request_worker.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +module Deployments + class LinkMergeRequestWorker + include ApplicationWorker + + queue_namespace :deployment + idempotent! + feature_category :continuous_delivery + worker_resource_boundary :cpu + + def perform(deployment_id) + if (deploy = Deployment.find_by_id(deployment_id)) + LinkMergeRequestsService.new(deploy).execute + end + end + end +end diff --git a/app/workers/deployments/success_worker.rb b/app/workers/deployments/success_worker.rb index 17f790d2f6f..b72b107985b 100644 --- a/app/workers/deployments/success_worker.rb +++ b/app/workers/deployments/success_worker.rb @@ -1,5 +1,7 @@ # frozen_string_literal: true +# This worker is deprecated and will be removed in 14.0 +# See: https://gitlab.com/gitlab-org/gitlab/-/issues/266381 module Deployments class SuccessWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker @@ -12,7 +14,7 @@ module Deployments Deployment.find_by_id(deployment_id).try do |deployment| break unless deployment.success? - Deployments::AfterCreateService.new(deployment).execute + Deployments::UpdateEnvironmentService.new(deployment).execute end end end diff --git a/app/workers/deployments/update_environment_worker.rb b/app/workers/deployments/update_environment_worker.rb new file mode 100644 index 00000000000..2381f9926bc --- /dev/null +++ b/app/workers/deployments/update_environment_worker.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +module Deployments + class UpdateEnvironmentWorker + include ApplicationWorker + + queue_namespace :deployment + idempotent! + feature_category :continuous_delivery + worker_resource_boundary :cpu + + def perform(deployment_id) + Deployment.find_by_id(deployment_id).try do |deployment| + break unless deployment.success? + + Deployments::UpdateEnvironmentService.new(deployment).execute + end + end + end +end diff --git a/app/workers/design_management/copy_design_collection_worker.rb b/app/workers/design_management/copy_design_collection_worker.rb new file mode 100644 index 00000000000..0a6e23fe9da --- /dev/null +++ b/app/workers/design_management/copy_design_collection_worker.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +module DesignManagement + class CopyDesignCollectionWorker + include ApplicationWorker + + feature_category :design_management + idempotent! + urgency :low + + def perform(user_id, issue_id, target_issue_id) + user = User.find(user_id) + issue = Issue.find(issue_id) + target_issue = Issue.find(target_issue_id) + + response = DesignManagement::CopyDesignCollection::CopyService.new( + target_issue.project, + user, + issue: issue, + target_issue: target_issue + ).execute + + Gitlab::AppLogger.warn(response.message) if response.error? + end + end +end diff --git a/app/workers/design_management/new_version_worker.rb b/app/workers/design_management/new_version_worker.rb index 3634dcbcebd..4fbf2067be4 100644 --- a/app/workers/design_management/new_version_worker.rb +++ b/app/workers/design_management/new_version_worker.rb @@ -9,10 +9,10 @@ module DesignManagement # `GenerateImageVersionsService` resizing designs worker_resource_boundary :memory - def perform(version_id) + def perform(version_id, skip_system_notes = false) version = DesignManagement::Version.find(version_id) - add_system_note(version) + add_system_note(version) unless skip_system_notes generate_image_versions(version) rescue ActiveRecord::RecordNotFound => e Sidekiq.logger.warn(e) diff --git a/app/workers/disallow_two_factor_for_group_worker.rb b/app/workers/disallow_two_factor_for_group_worker.rb new file mode 100644 index 00000000000..b3cc7a44672 --- /dev/null +++ b/app/workers/disallow_two_factor_for_group_worker.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +class DisallowTwoFactorForGroupWorker + include ApplicationWorker + include ExceptionBacktrace + + feature_category :subgroups + idempotent! + + def perform(group_id) + begin + group = Group.find(group_id) + rescue ActiveRecord::RecordNotFound + return + end + + group.update!(require_two_factor_authentication: false) + end +end diff --git a/app/workers/disallow_two_factor_for_subgroups_worker.rb b/app/workers/disallow_two_factor_for_subgroups_worker.rb new file mode 100644 index 00000000000..1ca227030e2 --- /dev/null +++ b/app/workers/disallow_two_factor_for_subgroups_worker.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +class DisallowTwoFactorForSubgroupsWorker + include ApplicationWorker + include ExceptionBacktrace + + INTERVAL = 2.seconds.to_i + + feature_category :subgroups + idempotent! + + def perform(group_id) + begin + group = Group.find(group_id) + rescue ActiveRecord::RecordNotFound + return + end + + # rubocop: disable CodeReuse/ActiveRecord + subgroups = group.descendants.where(require_two_factor_authentication: true) # rubocop: disable CodeReuse/ActiveRecord + subgroups.find_each(batch_size: 100).with_index do |subgroup, index| + delay = index * INTERVAL + + with_context(namespace: subgroup) do + DisallowTwoFactorForGroupWorker.perform_in(delay, subgroup.id) + end + end + # rubocop: enable CodeReuse/ActiveRecord + end +end diff --git a/app/workers/expire_build_instance_artifacts_worker.rb b/app/workers/expire_build_instance_artifacts_worker.rb index e6cd60a3e47..a5571473b43 100644 --- a/app/workers/expire_build_instance_artifacts_worker.rb +++ b/app/workers/expire_build_instance_artifacts_worker.rb @@ -4,6 +4,7 @@ class ExpireBuildInstanceArtifactsWorker # rubocop:disable Scalability/Idempoten include ApplicationWorker feature_category :continuous_integration + tags :requires_disk_io # rubocop: disable CodeReuse/ActiveRecord def perform(build_id) diff --git a/app/workers/export_csv_worker.rb b/app/workers/export_csv_worker.rb index e7baaf40a41..f2da381a34a 100644 --- a/app/workers/export_csv_worker.rb +++ b/app/workers/export_csv_worker.rb @@ -15,8 +15,6 @@ class ExportCsvWorker # rubocop:disable Scalability/IdempotentWorker params[:project_id] = project_id params.delete(:sort) - issues = IssuesFinder.new(@current_user, params).execute - - Issues::ExportCsvService.new(issues, @project).email(@current_user) + IssuableExportCsvWorker.perform_async(:issue, @current_user.id, @project.id, params) end end diff --git a/app/workers/git_garbage_collect_worker.rb b/app/workers/git_garbage_collect_worker.rb index b0307571448..9071e4b8a1b 100644 --- a/app/workers/git_garbage_collect_worker.rb +++ b/app/workers/git_garbage_collect_worker.rb @@ -91,14 +91,16 @@ class GitGarbageCollectWorker # rubocop:disable Scalability/IdempotentWorker end def cleanup_orphan_lfs_file_references(project) - return unless Feature.enabled?(:cleanup_lfs_during_gc, project) return if Gitlab::Database.read_only? # GitGarbageCollectWorker may be run on a Geo secondary ::Gitlab::Cleanup::OrphanLfsFileReferences.new(project, dry_run: false, logger: logger).run! + rescue => err + Gitlab::GitLogger.warn(message: "Cleaning up orphan LFS objects files failed", error: err.message) + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(err) end def flush_ref_caches(project) - project.repository.after_create_branch + project.repository.expire_branches_cache project.repository.branch_names project.repository.has_visible_content? end diff --git a/app/workers/group_destroy_worker.rb b/app/workers/group_destroy_worker.rb index d80a2dad7d9..901785f462b 100644 --- a/app/workers/group_destroy_worker.rb +++ b/app/workers/group_destroy_worker.rb @@ -5,6 +5,7 @@ class GroupDestroyWorker # rubocop:disable Scalability/IdempotentWorker include ExceptionBacktrace feature_category :subgroups + tags :requires_disk_io def perform(group_id, user_id) begin diff --git a/app/workers/group_export_worker.rb b/app/workers/group_export_worker.rb index e22b691d35e..a212147d8fd 100644 --- a/app/workers/group_export_worker.rb +++ b/app/workers/group_export_worker.rb @@ -6,7 +6,7 @@ class GroupExportWorker # rubocop:disable Scalability/IdempotentWorker feature_category :importers loggable_arguments 2 - sidekiq_options retry: false + sidekiq_options retry: false, dead: false def perform(current_user_id, group_id, params = {}) current_user = User.find(current_user_id) diff --git a/app/workers/group_import_worker.rb b/app/workers/group_import_worker.rb index 36d81468d55..b8b596f459b 100644 --- a/app/workers/group_import_worker.rb +++ b/app/workers/group_import_worker.rb @@ -3,13 +3,13 @@ class GroupImportWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker - sidekiq_options retry: false + sidekiq_options retry: false, dead: false feature_category :importers def perform(user_id, group_id) current_user = User.find(user_id) group = Group.find(group_id) - group_import_state = group.import_state || group.build_import_state + group_import_state = group.import_state group_import_state.jid = self.jid group_import_state.start! diff --git a/app/workers/incident_management/add_severity_system_note_worker.rb b/app/workers/incident_management/add_severity_system_note_worker.rb new file mode 100644 index 00000000000..9f132531562 --- /dev/null +++ b/app/workers/incident_management/add_severity_system_note_worker.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +module IncidentManagement + class AddSeveritySystemNoteWorker # rubocop:disable Scalability/IdempotentWorker + include ApplicationWorker + + queue_namespace :incident_management + feature_category :incident_management + + def perform(incident_id, user_id) + return if incident_id.blank? || user_id.blank? + + incident = Issue.with_issue_type(:incident).find_by_id(incident_id) + return unless incident + + user = User.find_by_id(user_id) + return unless user + + SystemNoteService.change_incident_severity(incident, user) + end + end +end diff --git a/app/workers/issuable_export_csv_worker.rb b/app/workers/issuable_export_csv_worker.rb new file mode 100644 index 00000000000..d91ba77287f --- /dev/null +++ b/app/workers/issuable_export_csv_worker.rb @@ -0,0 +1,53 @@ +# frozen_string_literal: true + +class IssuableExportCsvWorker # rubocop:disable Scalability/IdempotentWorker + include ApplicationWorker + + feature_category :issue_tracking + worker_resource_boundary :cpu + loggable_arguments 2 + + PERMITTED_TYPES = [:merge_request, :issue].freeze + + def perform(type, current_user_id, project_id, params) + @type = type.to_sym + check_permitted_type! + process_params!(params, project_id) + + @current_user = User.find(current_user_id) + @project = Project.find(project_id) + @service = service(find_objects(params)) + + @service.email(@current_user) + end + + private + + def find_objects(params) + case @type + when :issue + IssuesFinder.new(@current_user, params).execute + when :merge_request + MergeRequestsFinder.new(@current_user, params).execute + end + end + + def service(issuables) + case @type + when :issue + Issues::ExportCsvService.new(issuables, @project) + when :merge_request + MergeRequests::ExportCsvService.new(issuables, @project) + end + end + + def process_params!(params, project_id) + params.symbolize_keys! + params[:project_id] = project_id + params.delete(:sort) + end + + def check_permitted_type! + raise ArgumentError, "type parameter must be :issue or :merge_request, it was #{@type}" unless PERMITTED_TYPES.include?(@type) + end +end diff --git a/app/workers/issue_placement_worker.rb b/app/workers/issue_placement_worker.rb index a8d59e9125c..5b547ab0c8d 100644 --- a/app/workers/issue_placement_worker.rb +++ b/app/workers/issue_placement_worker.rb @@ -36,14 +36,14 @@ class IssuePlacementWorker Gitlab::ErrorTracking.log_exception(e, issue_id: issue_id, project_id: project_id) IssueRebalancingWorker.perform_async(nil, project_id.presence || issue.project_id) end - # rubocop: enable CodeReuse/ActiveRecord def find_issue(issue_id, project_id) - return Issue.id_in(issue_id).first if issue_id + return Issue.id_in(issue_id).take if issue_id - project = Project.id_in(project_id).first + project = Project.id_in(project_id).take return unless project - project.issues.first + project.issues.take end + # rubocop: enable CodeReuse/ActiveRecord end diff --git a/app/workers/issue_rebalancing_worker.rb b/app/workers/issue_rebalancing_worker.rb index 032ba5534e6..a9ad66198f3 100644 --- a/app/workers/issue_rebalancing_worker.rb +++ b/app/workers/issue_rebalancing_worker.rb @@ -11,7 +11,8 @@ class IssueRebalancingWorker return if project_id.nil? project = Project.find(project_id) - issue = project.issues.first # All issues are equivalent as far as we are concerned + # All issues are equivalent as far as we are concerned + issue = project.issues.take # rubocop: disable CodeReuse/ActiveRecord IssueRebalancingService.new(issue).execute rescue ActiveRecord::RecordNotFound, IssueRebalancingService::TooManyIssues => e diff --git a/app/workers/member_invitation_reminder_emails_worker.rb b/app/workers/member_invitation_reminder_emails_worker.rb new file mode 100644 index 00000000000..50f583005c0 --- /dev/null +++ b/app/workers/member_invitation_reminder_emails_worker.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +class MemberInvitationReminderEmailsWorker # rubocop:disable Scalability/IdempotentWorker + include ApplicationWorker + include CronjobQueue # rubocop:disable Scalability/CronWorkerContext + + feature_category :subgroups + urgency :low + + def perform + return unless Gitlab::Experimentation.enabled?(:invitation_reminders) + + Member.not_accepted_invitations.not_expired.last_ten_days_excluding_today.find_in_batches do |invitations| + invitations.each do |invitation| + Members::InvitationReminderEmailService.new(invitation).execute + end + end + end +end diff --git a/app/workers/metrics/dashboard/sync_dashboards_worker.rb b/app/workers/metrics/dashboard/sync_dashboards_worker.rb new file mode 100644 index 00000000000..7a124a33f9e --- /dev/null +++ b/app/workers/metrics/dashboard/sync_dashboards_worker.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +module Metrics + module Dashboard + class SyncDashboardsWorker + include ApplicationWorker + + feature_category :metrics + + idempotent! + + def perform(project_id) + project = Project.find(project_id) + dashboard_paths = ::Gitlab::Metrics::Dashboard::RepoDashboardFinder.list_dashboards(project) + + dashboard_paths.each do |dashboard_path| + ::Gitlab::Metrics::Dashboard::Importer.new(dashboard_path, project).execute! + end + end + end + end +end diff --git a/app/workers/pages_domain_ssl_renewal_worker.rb b/app/workers/pages_domain_ssl_renewal_worker.rb index 561fd59d471..125ba343948 100644 --- a/app/workers/pages_domain_ssl_renewal_worker.rb +++ b/app/workers/pages_domain_ssl_renewal_worker.rb @@ -4,6 +4,7 @@ class PagesDomainSslRenewalWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker feature_category :pages + tags :requires_disk_io def perform(domain_id) domain = PagesDomain.find_by_id(domain_id) diff --git a/app/workers/pages_domain_verification_worker.rb b/app/workers/pages_domain_verification_worker.rb index 1b4d9d3994c..ff0463481cd 100644 --- a/app/workers/pages_domain_verification_worker.rb +++ b/app/workers/pages_domain_verification_worker.rb @@ -4,6 +4,7 @@ class PagesDomainVerificationWorker # rubocop:disable Scalability/IdempotentWork include ApplicationWorker feature_category :pages + tags :requires_disk_io # rubocop: disable CodeReuse/ActiveRecord def perform(domain_id) diff --git a/app/workers/pages_worker.rb b/app/workers/pages_worker.rb index aefa4bc4223..0c561626f8c 100644 --- a/app/workers/pages_worker.rb +++ b/app/workers/pages_worker.rb @@ -6,6 +6,7 @@ class PagesWorker # rubocop:disable Scalability/IdempotentWorker sidekiq_options retry: 3 feature_category :pages loggable_arguments 0, 1 + tags :requires_disk_io def perform(action, *arg) send(action, *arg) # rubocop:disable GitlabSecurity/PublicSend diff --git a/app/workers/post_receive.rb b/app/workers/post_receive.rb index 4a93b1af166..0b224b88e4d 100644 --- a/app/workers/post_receive.rb +++ b/app/workers/post_receive.rb @@ -24,7 +24,7 @@ class PostReceive # rubocop:disable Scalability/IdempotentWorker post_received = Gitlab::GitPostReceive.new(container, identifier, changes, push_options) if repo_type.wiki? - process_wiki_changes(post_received, container.wiki) + process_wiki_changes(post_received, container) elsif repo_type.project? process_project_changes(post_received, container) elsif repo_type.snippet? diff --git a/app/workers/project_destroy_worker.rb b/app/workers/project_destroy_worker.rb index b3e7996f4a4..99d51fc5c2e 100644 --- a/app/workers/project_destroy_worker.rb +++ b/app/workers/project_destroy_worker.rb @@ -5,6 +5,7 @@ class ProjectDestroyWorker # rubocop:disable Scalability/IdempotentWorker include ExceptionBacktrace feature_category :source_code_management + tags :requires_disk_io def perform(project_id, user_id, params) project = Project.find(project_id) diff --git a/app/workers/project_export_worker.rb b/app/workers/project_export_worker.rb index 6c8640138a1..1c4aa3f7e49 100644 --- a/app/workers/project_export_worker.rb +++ b/app/workers/project_export_worker.rb @@ -8,7 +8,7 @@ class ProjectExportWorker # rubocop:disable Scalability/IdempotentWorker worker_resource_boundary :memory urgency :throttled loggable_arguments 2, 3 - sidekiq_options retry: false + sidekiq_options retry: false, dead: false sidekiq_options status_expiration: StuckExportJobsWorker::EXPORT_JOBS_EXPIRATION def perform(current_user_id, project_id, after_export_strategy = {}, params = {}) diff --git a/app/workers/propagate_integration_group_worker.rb b/app/workers/propagate_integration_group_worker.rb new file mode 100644 index 00000000000..01155753877 --- /dev/null +++ b/app/workers/propagate_integration_group_worker.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +class PropagateIntegrationGroupWorker + include ApplicationWorker + + feature_category :integrations + idempotent! + + # rubocop: disable CodeReuse/ActiveRecord + def perform(integration_id, min_id, max_id) + integration = Service.find_by_id(integration_id) + return unless integration + + batch = if integration.instance? + Group.where(id: min_id..max_id).without_integration(integration) + else + integration.group.descendants.where(id: min_id..max_id).without_integration(integration) + end + + return if batch.empty? + + BulkCreateIntegrationService.new(integration, batch, 'group').execute + end + # rubocop: enable CodeReuse/ActiveRecord +end diff --git a/app/workers/propagate_integration_inherit_worker.rb b/app/workers/propagate_integration_inherit_worker.rb new file mode 100644 index 00000000000..ef3132202f6 --- /dev/null +++ b/app/workers/propagate_integration_inherit_worker.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +class PropagateIntegrationInheritWorker + include ApplicationWorker + + feature_category :integrations + idempotent! + + # rubocop: disable CodeReuse/ActiveRecord + def perform(integration_id, min_id, max_id) + integration = Service.find_by_id(integration_id) + return unless integration + + services = Service.where(id: min_id..max_id).by_type(integration.type).inherit_from_id(integration.id) + + BulkUpdateIntegrationService.new(integration, services).execute + end + # rubocop: enable CodeReuse/ActiveRecord +end diff --git a/app/workers/propagate_integration_project_worker.rb b/app/workers/propagate_integration_project_worker.rb new file mode 100644 index 00000000000..188d81e5fc1 --- /dev/null +++ b/app/workers/propagate_integration_project_worker.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +class PropagateIntegrationProjectWorker + include ApplicationWorker + + feature_category :integrations + idempotent! + + # rubocop: disable CodeReuse/ActiveRecord + def perform(integration_id, min_id, max_id) + integration = Service.find_by_id(integration_id) + return unless integration + + batch = Project.where(id: min_id..max_id).without_integration(integration) + batch = batch.in_namespace(integration.group.self_and_descendants) if integration.group_id + + return if batch.empty? + + BulkCreateIntegrationService.new(integration, batch, 'project').execute + end + # rubocop: enable CodeReuse/ActiveRecord +end diff --git a/app/workers/propagate_integration_worker.rb b/app/workers/propagate_integration_worker.rb index 68e38386372..bb954b12a25 100644 --- a/app/workers/propagate_integration_worker.rb +++ b/app/workers/propagate_integration_worker.rb @@ -7,7 +7,8 @@ class PropagateIntegrationWorker idempotent! loggable_arguments 1 - # Keep overwrite parameter for backwards compatibility. + # TODO: Keep overwrite parameter for backwards compatibility. Remove after >= 14.0 + # https://gitlab.com/gitlab-org/gitlab/-/issues/255382 def perform(integration_id, overwrite = nil) Admin::PropagateIntegrationService.propagate(Service.find(integration_id)) end diff --git a/app/workers/repository_import_worker.rb b/app/workers/repository_import_worker.rb index 54052bda675..51fe60e25fc 100644 --- a/app/workers/repository_import_worker.rb +++ b/app/workers/repository_import_worker.rb @@ -7,7 +7,8 @@ class RepositoryImportWorker # rubocop:disable Scalability/IdempotentWorker feature_category :importers worker_has_external_dependencies! - sidekiq_options retry: false + # Do not retry on Import/Export until https://gitlab.com/gitlab-org/gitlab/-/issues/16812 is solved. + sidekiq_options retry: false, dead: false sidekiq_options status_expiration: Gitlab::Import::StuckImportJob::IMPORT_JOBS_EXPIRATION # technical debt: https://gitlab.com/gitlab-org/gitlab/issues/33991 diff --git a/app/workers/stuck_ci_jobs_worker.rb b/app/workers/stuck_ci_jobs_worker.rb index f8209ae5e63..eca0a248a37 100644 --- a/app/workers/stuck_ci_jobs_worker.rb +++ b/app/workers/stuck_ci_jobs_worker.rb @@ -17,7 +17,7 @@ class StuckCiJobsWorker # rubocop:disable Scalability/IdempotentWorker def perform return unless try_obtain_lease - Gitlab::AppLogger.info "#{self.class}: Cleaning stuck builds" # rubocop:disable Gitlab/RailsLogger + Gitlab::AppLogger.info "#{self.class}: Cleaning stuck builds" drop :running, BUILD_RUNNING_OUTDATED_TIMEOUT, 'ci_builds.updated_at < ?', :stuck_or_timeout_failure drop :pending, BUILD_PENDING_OUTDATED_TIMEOUT, 'ci_builds.updated_at < ?', :stuck_or_timeout_failure diff --git a/app/workers/web_hooks/destroy_worker.rb b/app/workers/web_hooks/destroy_worker.rb new file mode 100644 index 00000000000..13a5a7bf1e6 --- /dev/null +++ b/app/workers/web_hooks/destroy_worker.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +module WebHooks + class DestroyWorker + include ApplicationWorker + + feature_category :integrations + urgency :low + idempotent! + + def perform(user_id, web_hook_id) + user = User.find_by_id(user_id) + hook = WebHook.find_by_id(web_hook_id) + + return unless user && hook + + result = ::WebHooks::DestroyService.new(user).sync_destroy(hook) + + return result if result[:status] == :success + + e = ::WebHooks::DestroyService::DestroyError.new(result[:message]) + Gitlab::ErrorTracking.track_exception(e, web_hook_id: hook.id) + + raise e + end + end +end |