diff options
Diffstat (limited to 'app/workers')
33 files changed, 487 insertions, 126 deletions
diff --git a/app/workers/all_queues.yml b/app/workers/all_queues.yml index 59ab0a4d05b..ff26aa7a4be 100644 --- a/app/workers/all_queues.yml +++ b/app/workers/all_queues.yml @@ -131,6 +131,14 @@ :weight: 1 :idempotent: true :tags: [] +- :name: cronjob:analytics_usage_trends_count_job_trigger + :feature_category: :devops_reports + :has_external_dependencies: + :urgency: :low + :resource_boundary: :unknown + :weight: 1 + :idempotent: true + :tags: [] - :name: cronjob:authorized_project_update_periodic_recalculate :feature_category: :source_code_management :has_external_dependencies: @@ -347,14 +355,6 @@ :weight: 1 :idempotent: :tags: [] -- :name: cronjob:releases_create_evidence - :feature_category: :release_evidence - :has_external_dependencies: - :urgency: :low - :resource_boundary: :unknown - :weight: 1 - :idempotent: - :tags: [] - :name: cronjob:releases_manage_evidence :feature_category: :release_evidence :has_external_dependencies: @@ -1083,6 +1083,14 @@ :weight: 1 :idempotent: :tags: [] +- :name: package_repositories:packages_maven_metadata_sync + :feature_category: :package_registry + :has_external_dependencies: + :urgency: :low + :resource_boundary: :unknown + :weight: 1 + :idempotent: true + :tags: [] - :name: package_repositories:packages_nuget_extraction :feature_category: :package_registry :has_external_dependencies: @@ -1098,8 +1106,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: - :tags: - - :requires_disk_io + :tags: [] - :name: pipeline_background:ci_build_trace_chunk_flush :feature_category: :continuous_integration :has_external_dependencies: @@ -1243,8 +1250,7 @@ :resource_boundary: :cpu :weight: 5 :idempotent: - :tags: - - :requires_disk_io + :tags: [] - :name: pipeline_processing:build_queue :feature_category: :continuous_integration :has_external_dependencies: @@ -1413,6 +1419,14 @@ :weight: 1 :idempotent: true :tags: [] +- :name: analytics_usage_trends_counter_job + :feature_category: :devops_reports + :has_external_dependencies: + :urgency: :low + :resource_boundary: :unknown + :weight: 1 + :idempotent: true + :tags: [] - :name: approve_blocked_pending_approval_users :feature_category: :users :has_external_dependencies: @@ -1468,8 +1482,7 @@ :resource_boundary: :unknown :weight: 2 :idempotent: - :tags: - - :requires_disk_io + :tags: [] - :name: ci_delete_objects :feature_category: :continuous_integration :has_external_dependencies: @@ -1629,8 +1642,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: - :tags: - - :requires_disk_io + :tags: [] - :name: export_csv :feature_category: :issue_tracking :has_external_dependencies: @@ -1800,6 +1812,14 @@ :weight: 1 :idempotent: true :tags: [] +- :name: merge_requests_delete_source_branch + :feature_category: :source_code_management + :has_external_dependencies: + :urgency: :high + :resource_boundary: :unknown + :weight: 1 + :idempotent: true + :tags: [] - :name: metrics_dashboard_prune_old_annotations :feature_category: :metrics :has_external_dependencies: @@ -1832,6 +1852,14 @@ :weight: 1 :idempotent: :tags: [] +- :name: namespaces_onboarding_issue_created + :feature_category: :issue_tracking + :has_external_dependencies: + :urgency: :low + :resource_boundary: :unknown + :weight: 1 + :idempotent: true + :tags: [] - :name: namespaces_onboarding_pipeline_created :feature_category: :subgroups :has_external_dependencies: @@ -1840,6 +1868,14 @@ :weight: 1 :idempotent: true :tags: [] +- :name: namespaces_onboarding_progress + :feature_category: :product_analytics + :has_external_dependencies: + :urgency: :low + :resource_boundary: :unknown + :weight: 1 + :idempotent: true + :tags: [] - :name: namespaces_onboarding_user_added :feature_category: :users :has_external_dependencies: @@ -1872,6 +1908,14 @@ :weight: 2 :idempotent: :tags: [] +- :name: packages_composer_cache_update + :feature_category: :package_registry + :has_external_dependencies: + :urgency: :low + :resource_boundary: :unknown + :weight: 1 + :idempotent: true + :tags: [] - :name: pages :feature_category: :pages :has_external_dependencies: @@ -2012,6 +2056,22 @@ :weight: 1 :idempotent: :tags: [] +- :name: projects_schedule_bulk_repository_shard_moves + :feature_category: :gitaly + :has_external_dependencies: + :urgency: :throttled + :resource_boundary: :unknown + :weight: 1 + :idempotent: true + :tags: [] +- :name: projects_update_repository_storage + :feature_category: :gitaly + :has_external_dependencies: + :urgency: :throttled + :resource_boundary: :unknown + :weight: 1 + :idempotent: true + :tags: [] - :name: prometheus_create_default_alerts :feature_category: :incident_management :has_external_dependencies: @@ -2084,6 +2144,14 @@ :weight: 2 :idempotent: :tags: [] +- :name: releases_create_evidence + :feature_category: :release_evidence + :has_external_dependencies: + :urgency: :low + :resource_boundary: :unknown + :weight: 1 + :idempotent: + :tags: [] - :name: remote_mirror_notification :feature_category: :source_code_management :has_external_dependencies: @@ -2172,6 +2240,22 @@ :weight: 1 :idempotent: true :tags: [] +- :name: snippets_schedule_bulk_repository_shard_moves + :feature_category: :gitaly + :has_external_dependencies: + :urgency: :throttled + :resource_boundary: :unknown + :weight: 1 + :idempotent: true + :tags: [] +- :name: snippets_update_repository_storage + :feature_category: :gitaly + :has_external_dependencies: + :urgency: :throttled + :resource_boundary: :unknown + :weight: 1 + :idempotent: true + :tags: [] - :name: system_hook_push :feature_category: :source_code_management :has_external_dependencies: diff --git a/app/workers/analytics/instance_statistics/count_job_trigger_worker.rb b/app/workers/analytics/instance_statistics/count_job_trigger_worker.rb index 81a765d5d08..3ec92bc7635 100644 --- a/app/workers/analytics/instance_statistics/count_job_trigger_worker.rb +++ b/app/workers/analytics/instance_statistics/count_job_trigger_worker.rb @@ -2,31 +2,19 @@ module Analytics module InstanceStatistics + # This worker will be removed in 14.0 class CountJobTriggerWorker include ApplicationWorker include CronjobQueue # rubocop:disable Scalability/CronWorkerContext - DEFAULT_DELAY = 3.minutes.freeze - feature_category :devops_reports urgency :low idempotent! def perform - recorded_at = Time.zone.now - - worker_arguments = Gitlab::Analytics::InstanceStatistics::WorkersArgumentBuilder.new( - measurement_identifiers: ::Analytics::InstanceStatistics::Measurement.measurement_identifier_values, - recorded_at: recorded_at - ).execute - - perform_in = DEFAULT_DELAY.minutes.from_now - worker_arguments.each do |args| - CounterJobWorker.perform_in(perform_in, *args) - - perform_in += DEFAULT_DELAY - end + # Delegate to the new worker + Analytics::UsageTrends::CountJobTriggerWorker.new.perform end end end diff --git a/app/workers/analytics/instance_statistics/counter_job_worker.rb b/app/workers/analytics/instance_statistics/counter_job_worker.rb index c07b2569453..4beed8a3e2f 100644 --- a/app/workers/analytics/instance_statistics/counter_job_worker.rb +++ b/app/workers/analytics/instance_statistics/counter_job_worker.rb @@ -2,6 +2,7 @@ module Analytics module InstanceStatistics + # This worker will be removed in 14.0 class CounterJobWorker include ApplicationWorker @@ -10,24 +11,9 @@ module Analytics idempotent! - def perform(measurement_identifier, min_id, max_id, recorded_at) - query_scope = ::Analytics::InstanceStatistics::Measurement.identifier_query_mapping[measurement_identifier].call - - count = if min_id.nil? || max_id.nil? # table is empty - 0 - else - counter(query_scope, min_id, max_id) - end - - return if count == Gitlab::Database::BatchCounter::FALLBACK - - InstanceStatistics::Measurement.insert_all([{ recorded_at: recorded_at, count: count, identifier: measurement_identifier }]) - end - - private - - def counter(query_scope, min_id, max_id) - Gitlab::Database::BatchCount.batch_count(query_scope, start: min_id, finish: max_id) + def perform(*args) + # Delegate to the new worker + Analytics::UsageTrends::CounterJobWorker.new.perform(*args) end end end diff --git a/app/workers/analytics/usage_trends/count_job_trigger_worker.rb b/app/workers/analytics/usage_trends/count_job_trigger_worker.rb new file mode 100644 index 00000000000..37f5c19d64c --- /dev/null +++ b/app/workers/analytics/usage_trends/count_job_trigger_worker.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +module Analytics + module UsageTrends + class CountJobTriggerWorker + extend ::Gitlab::Utils::Override + include ApplicationWorker + include CronjobQueue # rubocop:disable Scalability/CronWorkerContext + + DEFAULT_DELAY = 3.minutes.freeze + + feature_category :devops_reports + urgency :low + + idempotent! + + def perform + recorded_at = Time.zone.now + + worker_arguments = Gitlab::Analytics::UsageTrends::WorkersArgumentBuilder.new( + measurement_identifiers: ::Analytics::UsageTrends::Measurement.measurement_identifier_values, + recorded_at: recorded_at + ).execute + + perform_in = DEFAULT_DELAY.minutes.from_now + worker_arguments.each do |args| + CounterJobWorker.perform_in(perform_in, *args) + + perform_in += DEFAULT_DELAY + end + end + end + end +end diff --git a/app/workers/analytics/usage_trends/counter_job_worker.rb b/app/workers/analytics/usage_trends/counter_job_worker.rb new file mode 100644 index 00000000000..275c6ac2de2 --- /dev/null +++ b/app/workers/analytics/usage_trends/counter_job_worker.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +module Analytics + module UsageTrends + class CounterJobWorker + extend ::Gitlab::Utils::Override + include ApplicationWorker + + feature_category :devops_reports + urgency :low + + idempotent! + + def perform(measurement_identifier, min_id, max_id, recorded_at) + query_scope = ::Analytics::UsageTrends::Measurement.identifier_query_mapping[measurement_identifier].call + + count = if min_id.nil? || max_id.nil? # table is empty + 0 + else + counter(query_scope, min_id, max_id) + end + + return if count == Gitlab::Database::BatchCounter::FALLBACK + + UsageTrends::Measurement.insert_all([{ recorded_at: recorded_at, count: count, identifier: measurement_identifier }]) + end + + private + + def counter(query_scope, min_id, max_id) + Gitlab::Database::BatchCount.batch_count(query_scope, start: min_id, finish: max_id) + end + end + end +end diff --git a/app/workers/archive_trace_worker.rb b/app/workers/archive_trace_worker.rb index b0c5bef336a..3ddb5686bf2 100644 --- a/app/workers/archive_trace_worker.rb +++ b/app/workers/archive_trace_worker.rb @@ -4,8 +4,6 @@ class ArchiveTraceWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker include PipelineBackgroundQueue - tags :requires_disk_io - # rubocop: disable CodeReuse/ActiveRecord def perform(job_id) Ci::Build.without_archived_trace.find_by(id: job_id).try do |job| diff --git a/app/workers/build_finished_worker.rb b/app/workers/build_finished_worker.rb index 4d15bcd16f7..3f99b30fdf7 100644 --- a/app/workers/build_finished_worker.rb +++ b/app/workers/build_finished_worker.rb @@ -7,7 +7,6 @@ class BuildFinishedWorker # rubocop:disable Scalability/IdempotentWorker queue_namespace :pipeline_processing urgency :high worker_resource_boundary :cpu - tags :requires_disk_io ARCHIVE_TRACES_IN = 2.minutes.freeze @@ -35,7 +34,7 @@ class BuildFinishedWorker # rubocop:disable Scalability/IdempotentWorker # We execute these async as these are independent operations. BuildHooksWorker.perform_async(build.id) - ExpirePipelineCacheWorker.perform_async(build.pipeline_id) if build.pipeline.cacheable? + ExpirePipelineCacheWorker.perform_async(build.pipeline_id) ChatNotificationWorker.perform_async(build.id) if build.pipeline.chat? ## diff --git a/app/workers/chat_notification_worker.rb b/app/workers/chat_notification_worker.rb index 94a0197b862..5fab437f49f 100644 --- a/app/workers/chat_notification_worker.rb +++ b/app/workers/chat_notification_worker.rb @@ -7,7 +7,6 @@ class ChatNotificationWorker # rubocop:disable Scalability/IdempotentWorker sidekiq_options retry: false feature_category :chatops - tags :requires_disk_io urgency :low # Can't be high as it has external dependencies weight 2 worker_has_external_dependencies! diff --git a/app/workers/concerns/worker_attributes.rb b/app/workers/concerns/worker_attributes.rb index bb6192166b4..042508d08f2 100644 --- a/app/workers/concerns/worker_attributes.rb +++ b/app/workers/concerns/worker_attributes.rb @@ -133,5 +133,13 @@ module WorkerAttributes def get_deduplication_options class_attributes[:deduplication_options] || {} end + + def big_payload! + class_attributes[:big_payload] = true + end + + def big_payload? + class_attributes[:big_payload] + end end end diff --git a/app/workers/emails_on_push_worker.rb b/app/workers/emails_on_push_worker.rb index 2f0d7fecf19..1a34bf50d87 100644 --- a/app/workers/emails_on_push_worker.rb +++ b/app/workers/emails_on_push_worker.rb @@ -96,6 +96,6 @@ class EmailsOnPushWorker # rubocop:disable Scalability/IdempotentWorker def valid_recipients(recipients) recipients.split.select do |recipient| recipient.include?('@') - end + end.uniq(&:downcase) end end diff --git a/app/workers/error_tracking_issue_link_worker.rb b/app/workers/error_tracking_issue_link_worker.rb index fa8af4f1822..4ad80d57f6b 100644 --- a/app/workers/error_tracking_issue_link_worker.rb +++ b/app/workers/error_tracking_issue_link_worker.rb @@ -26,7 +26,7 @@ class ErrorTrackingIssueLinkWorker # rubocop:disable Scalability/IdempotentWorke logger.info("Linking Sentry issue #{sentry_issue_id} to GitLab issue #{issue.id}") sentry_client.create_issue_link(integration_id, sentry_issue_id, issue) - rescue Sentry::Client::Error => e + rescue ErrorTracking::SentryClient::Error => e logger.info("Failed to link Sentry issue #{sentry_issue_id} to GitLab issue #{issue.id} with error: #{e.message}") end end @@ -63,7 +63,7 @@ class ErrorTrackingIssueLinkWorker # rubocop:disable Scalability/IdempotentWorke sentry_client .repos(organization_slug) .find { |repo| repo.project_id == issue.project_id && repo.status == 'active' } - rescue Sentry::Client::Error => e + rescue ErrorTracking::SentryClient::Error => e logger.info("Unable to retrieve Sentry repo for organization #{organization_slug}, id #{sentry_issue_id}, with error: #{e.message}") nil diff --git a/app/workers/expire_build_instance_artifacts_worker.rb b/app/workers/expire_build_instance_artifacts_worker.rb index a5571473b43..e6cd60a3e47 100644 --- a/app/workers/expire_build_instance_artifacts_worker.rb +++ b/app/workers/expire_build_instance_artifacts_worker.rb @@ -4,7 +4,6 @@ class ExpireBuildInstanceArtifactsWorker # rubocop:disable Scalability/Idempoten include ApplicationWorker feature_category :continuous_integration - tags :requires_disk_io # rubocop: disable CodeReuse/ActiveRecord def perform(build_id) diff --git a/app/workers/expire_job_cache_worker.rb b/app/workers/expire_job_cache_worker.rb index ce27fed7fb1..77b0edfd7de 100644 --- a/app/workers/expire_job_cache_worker.rb +++ b/app/workers/expire_job_cache_worker.rb @@ -16,19 +16,13 @@ class ExpireJobCacheWorker pipeline = job.pipeline project = job.project - Gitlab::EtagCaching::Store.new.tap do |store| - store.touch(project_pipeline_path(project, pipeline)) - store.touch(project_job_path(project, job)) - end + Gitlab::EtagCaching::Store.new.touch(project_job_path(project, job)) + ExpirePipelineCacheWorker.perform_async(pipeline.id) end # rubocop: enable CodeReuse/ActiveRecord private - def project_pipeline_path(project, pipeline) - Gitlab::Routing.url_helpers.project_pipeline_path(project, pipeline, format: :json) - end - def project_job_path(project, job) Gitlab::Routing.url_helpers.project_build_path(project, job.id, format: :json) end diff --git a/app/workers/expire_pipeline_cache_worker.rb b/app/workers/expire_pipeline_cache_worker.rb index 0710ef9298b..02039e40e15 100644 --- a/app/workers/expire_pipeline_cache_worker.rb +++ b/app/workers/expire_pipeline_cache_worker.rb @@ -13,7 +13,7 @@ class ExpirePipelineCacheWorker # rubocop: disable CodeReuse/ActiveRecord def perform(pipeline_id) pipeline = Ci::Pipeline.find_by(id: pipeline_id) - return unless pipeline&.cacheable? + return unless pipeline Ci::ExpirePipelineCacheService.new.execute(pipeline) end diff --git a/app/workers/merge_requests/delete_source_branch_worker.rb b/app/workers/merge_requests/delete_source_branch_worker.rb new file mode 100644 index 00000000000..eb83d10af33 --- /dev/null +++ b/app/workers/merge_requests/delete_source_branch_worker.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +class MergeRequests::DeleteSourceBranchWorker + include ApplicationWorker + + feature_category :source_code_management + urgency :high + idempotent! + + def perform(merge_request_id, source_branch_sha, user_id) + merge_request = MergeRequest.find(merge_request_id) + user = User.find(user_id) + + # Source branch changed while it's being removed + return if merge_request.source_branch_sha != source_branch_sha + + ::Branches::DeleteService.new(merge_request.source_project, user) + .execute(merge_request.source_branch) + + ::MergeRequests::RetargetChainService.new(merge_request.source_project, user) + .execute(merge_request) + rescue ActiveRecord::RecordNotFound + end +end diff --git a/app/workers/namespaces/in_product_marketing_emails_worker.rb b/app/workers/namespaces/in_product_marketing_emails_worker.rb index 66d140928a7..f8fa393264a 100644 --- a/app/workers/namespaces/in_product_marketing_emails_worker.rb +++ b/app/workers/namespaces/in_product_marketing_emails_worker.rb @@ -9,6 +9,7 @@ module Namespaces urgency :low def perform + return unless Gitlab::CurrentSettings.in_product_marketing_emails_enabled return unless Gitlab::Experimentation.active?(:in_product_marketing_emails) Namespaces::InProductMarketingEmailsService.send_for_all_tracks_and_intervals diff --git a/app/workers/namespaces/onboarding_issue_created_worker.rb b/app/workers/namespaces/onboarding_issue_created_worker.rb new file mode 100644 index 00000000000..e5e2c80e821 --- /dev/null +++ b/app/workers/namespaces/onboarding_issue_created_worker.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +module Namespaces + class OnboardingIssueCreatedWorker + include ApplicationWorker + + feature_category :issue_tracking + urgency :low + + deduplicate :until_executing + idempotent! + + def perform(namespace_id) + namespace = Namespace.find_by_id(namespace_id) + return unless namespace + + OnboardingProgressService.new(namespace).execute(action: :issue_created) + end + end +end diff --git a/app/workers/namespaces/onboarding_progress_worker.rb b/app/workers/namespaces/onboarding_progress_worker.rb new file mode 100644 index 00000000000..9cb5a23cf31 --- /dev/null +++ b/app/workers/namespaces/onboarding_progress_worker.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +module Namespaces + class OnboardingProgressWorker + include ApplicationWorker + + feature_category :product_analytics + urgency :low + + deduplicate :until_executed + idempotent! + + def perform(namespace_id, action) + namespace = Namespace.find_by_id(namespace_id) + return unless namespace && action + + OnboardingProgressService.new(namespace).execute(action: action.to_sym) + end + end +end diff --git a/app/workers/packages/composer/cache_update_worker.rb b/app/workers/packages/composer/cache_update_worker.rb new file mode 100644 index 00000000000..664fb23284f --- /dev/null +++ b/app/workers/packages/composer/cache_update_worker.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module Packages + module Composer + class CacheUpdateWorker + include ApplicationWorker + + feature_category :package_registry + + idempotent! + + def perform(project_id, package_name, last_page_sha) + project = Project.find_by_id(project_id) + + return unless project + + Gitlab::Composer::Cache.new(project: project, name: package_name, last_page_sha: last_page_sha).execute + rescue => e + Gitlab::ErrorTracking.log_exception(e, project_id: project_id) + end + end + end +end diff --git a/app/workers/packages/maven/metadata/sync_worker.rb b/app/workers/packages/maven/metadata/sync_worker.rb new file mode 100644 index 00000000000..eb7abf4cdd0 --- /dev/null +++ b/app/workers/packages/maven/metadata/sync_worker.rb @@ -0,0 +1,59 @@ +# frozen_string_literal: true + +module Packages + module Maven + module Metadata + class SyncWorker + include ApplicationWorker + include Gitlab::Utils::StrongMemoize + + queue_namespace :package_repositories + feature_category :package_registry + + deduplicate :until_executing + idempotent! + + loggable_arguments 2 + + SyncError = Class.new(StandardError) + + def perform(user_id, project_id, package_name) + @user_id = user_id + @project_id = project_id + @package_name = package_name + + return unless valid? + + result = ::Packages::Maven::Metadata::SyncService.new(container: project, current_user: user, params: { package_name: @package_name }) + .execute + + if result.success? + log_extra_metadata_on_done(:message, result.message) + else + raise SyncError.new(result.message) + end + + raise SyncError.new(result.message) unless result.success? + end + + private + + def valid? + @package_name.present? && user.present? && project.present? + end + + def user + strong_memoize(:user) do + User.find_by_id(@user_id) + end + end + + def project + strong_memoize(:project) do + Project.find_by_id(@project_id) + end + end + end + end + end +end diff --git a/app/workers/pages_update_configuration_worker.rb b/app/workers/pages_update_configuration_worker.rb index 07238bae8c2..6e82e2099c7 100644 --- a/app/workers/pages_update_configuration_worker.rb +++ b/app/workers/pages_update_configuration_worker.rb @@ -6,6 +6,12 @@ class PagesUpdateConfigurationWorker idempotent! feature_category :pages + def self.perform_async(*args) + return unless Feature.enabled?(:pages_update_legacy_storage, default_enabled: true) + + super(*args) + end + def perform(project_id) project = Project.find_by_id(project_id) return unless project diff --git a/app/workers/personal_access_tokens/expiring_worker.rb b/app/workers/personal_access_tokens/expiring_worker.rb index f9940d9d014..7a016c85a64 100644 --- a/app/workers/personal_access_tokens/expiring_worker.rb +++ b/app/workers/personal_access_tokens/expiring_worker.rb @@ -7,17 +7,32 @@ module PersonalAccessTokens feature_category :authentication_and_authorization + MAX_TOKENS = 100 + def perform(*args) notification_service = NotificationService.new limit_date = PersonalAccessToken::DAYS_TO_EXPIRE.days.from_now.to_date User.with_expiring_and_not_notified_personal_access_tokens(limit_date).find_each do |user| with_context(user: user) do - notification_service.access_token_about_to_expire(user) + expiring_user_tokens = user.personal_access_tokens.without_impersonation.expiring_and_not_notified(limit_date) + + # rubocop: disable CodeReuse/ActiveRecord + # We never materialise the token instances. We need the names to mention them in the + # email. Later we trigger an update query on the entire relation, not on individual instances. + token_names = expiring_user_tokens.limit(MAX_TOKENS).pluck(:name) + # We're limiting to 100 tokens so we avoid loading too many tokens into memory. + # At the time of writing this would only affect 69 users on GitLab.com + + # rubocop: enable CodeReuse/ActiveRecord + + notification_service.access_token_about_to_expire(user, token_names) Gitlab::AppLogger.info "#{self.class}: Notifying User #{user.id} about expiring tokens" - user.personal_access_tokens.without_impersonation.expiring_and_not_notified(limit_date).update_all(expire_notification_delivered: true) + expiring_user_tokens.each_batch do |expiring_tokens| + expiring_tokens.update_all(expire_notification_delivered: true) + end end end end diff --git a/app/workers/post_receive.rb b/app/workers/post_receive.rb index ac55f883fc5..313b901c08e 100644 --- a/app/workers/post_receive.rb +++ b/app/workers/post_receive.rb @@ -123,6 +123,7 @@ class PostReceive # rubocop:disable Scalability/IdempotentWorker def after_project_changes_hooks(project, user, refs, changes) experiment(:new_project_readme, actor: user).track_initial_writes(project) + experiment(:empty_repo_upload, project: project).track(:initial_write) if project.empty_repo? repository_update_hook_data = Gitlab::DataBuilder::Repository.update(project, user, changes, refs) SystemHooksService.new.execute_hooks(repository_update_hook_data, :repository_update_hooks) Gitlab::UsageDataCounters::SourceCodeCounter.count(:pushes) diff --git a/app/workers/project_schedule_bulk_repository_shard_moves_worker.rb b/app/workers/project_schedule_bulk_repository_shard_moves_worker.rb index 4d2a6b47e3c..23d1594e4d9 100644 --- a/app/workers/project_schedule_bulk_repository_shard_moves_worker.rb +++ b/app/workers/project_schedule_bulk_repository_shard_moves_worker.rb @@ -1,13 +1,15 @@ # frozen_string_literal: true -class ProjectScheduleBulkRepositoryShardMovesWorker - include ApplicationWorker - +# This is a compatibility class to avoid calling a non-existent +# class from sidekiq during deployment. +# +# This class was moved to a namespace in https://gitlab.com/gitlab-org/gitlab/-/issues/299853. +# we cannot remove this class entirely because there can be jobs +# referencing it. +# +# We can get rid of this class in 14.0 +# https://gitlab.com/gitlab-org/gitlab/-/issues/322393 +class ProjectScheduleBulkRepositoryShardMovesWorker < Projects::ScheduleBulkRepositoryShardMovesWorker idempotent! - feature_category :gitaly urgency :throttled - - def perform(source_storage_name, destination_storage_name = nil) - Projects::ScheduleBulkRepositoryShardMovesService.new.execute(source_storage_name, destination_storage_name) - end end diff --git a/app/workers/project_update_repository_storage_worker.rb b/app/workers/project_update_repository_storage_worker.rb index 5636eec8233..0d68c0e16f8 100644 --- a/app/workers/project_update_repository_storage_worker.rb +++ b/app/workers/project_update_repository_storage_worker.rb @@ -1,23 +1,15 @@ # frozen_string_literal: true -class ProjectUpdateRepositoryStorageWorker # rubocop:disable Scalability/IdempotentWorker - extend ::Gitlab::Utils::Override - include UpdateRepositoryStorageWorker - - private - - override :find_repository_storage_move - def find_repository_storage_move(repository_storage_move_id) - ProjectRepositoryStorageMove.find(repository_storage_move_id) - end - - override :find_container - def find_container(container_id) - Project.find(container_id) - end - - override :update_repository_storage - def update_repository_storage(repository_storage_move) - ::Projects::UpdateRepositoryStorageService.new(repository_storage_move).execute - end +# This is a compatibility class to avoid calling a non-existent +# class from sidekiq during deployment. +# +# This class was moved to a namespace in https://gitlab.com/gitlab-org/gitlab/-/issues/299853. +# we cannot remove this class entirely because there can be jobs +# referencing it. +# +# We can get rid of this class in 14.0 +# https://gitlab.com/gitlab-org/gitlab/-/issues/322393 +class ProjectUpdateRepositoryStorageWorker < Projects::UpdateRepositoryStorageWorker + idempotent! + urgency :throttled end diff --git a/app/workers/projects/schedule_bulk_repository_shard_moves_worker.rb b/app/workers/projects/schedule_bulk_repository_shard_moves_worker.rb new file mode 100644 index 00000000000..3841ae9b922 --- /dev/null +++ b/app/workers/projects/schedule_bulk_repository_shard_moves_worker.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Projects + class ScheduleBulkRepositoryShardMovesWorker + include ApplicationWorker + + idempotent! + feature_category :gitaly + urgency :throttled + + def perform(source_storage_name, destination_storage_name = nil) + Projects::ScheduleBulkRepositoryShardMovesService.new.execute(source_storage_name, destination_storage_name) + end + end +end diff --git a/app/workers/projects/update_repository_storage_worker.rb b/app/workers/projects/update_repository_storage_worker.rb new file mode 100644 index 00000000000..f4c44458446 --- /dev/null +++ b/app/workers/projects/update_repository_storage_worker.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module Projects + class UpdateRepositoryStorageWorker # rubocop:disable Scalability/IdempotentWorker + extend ::Gitlab::Utils::Override + include ::UpdateRepositoryStorageWorker + + private + + override :find_repository_storage_move + def find_repository_storage_move(repository_storage_move_id) + ::Projects::RepositoryStorageMove.find(repository_storage_move_id) + end + + override :find_container + def find_container(container_id) + Project.find(container_id) + end + + override :update_repository_storage + def update_repository_storage(repository_storage_move) + ::Projects::UpdateRepositoryStorageService.new(repository_storage_move).execute + end + end +end diff --git a/app/workers/releases/create_evidence_worker.rb b/app/workers/releases/create_evidence_worker.rb index db75fae1639..d22329216f9 100644 --- a/app/workers/releases/create_evidence_worker.rb +++ b/app/workers/releases/create_evidence_worker.rb @@ -3,7 +3,6 @@ module Releases class CreateEvidenceWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker - include CronjobQueue # rubocop:disable Scalability/CronWorkerContext feature_category :release_evidence diff --git a/app/workers/snippet_schedule_bulk_repository_shard_moves_worker.rb b/app/workers/snippet_schedule_bulk_repository_shard_moves_worker.rb index 47f24ad3500..94a6b22538b 100644 --- a/app/workers/snippet_schedule_bulk_repository_shard_moves_worker.rb +++ b/app/workers/snippet_schedule_bulk_repository_shard_moves_worker.rb @@ -1,13 +1,16 @@ # frozen_string_literal: true -class SnippetScheduleBulkRepositoryShardMovesWorker - include ApplicationWorker - +# This is a compatibility class to avoid calling a non-existent +# class from sidekiq during deployment. +# +# This class was moved to a namespace in https://gitlab.com/gitlab-org/gitlab/-/issues/299853. +# we cannot remove this class entirely because there can be jobs +# referencing it. +# +# We can get rid of this class in 14.0 +# https://gitlab.com/gitlab-org/gitlab/-/issues/322393 +class SnippetScheduleBulkRepositoryShardMovesWorker < Snippets::ScheduleBulkRepositoryShardMovesWorker idempotent! feature_category :gitaly urgency :throttled - - def perform(source_storage_name, destination_storage_name = nil) - Snippets::ScheduleBulkRepositoryShardMovesService.new.execute(source_storage_name, destination_storage_name) - end end diff --git a/app/workers/snippet_update_repository_storage_worker.rb b/app/workers/snippet_update_repository_storage_worker.rb index a28a02a0298..befae6db4f4 100644 --- a/app/workers/snippet_update_repository_storage_worker.rb +++ b/app/workers/snippet_update_repository_storage_worker.rb @@ -1,23 +1,15 @@ # frozen_string_literal: true -class SnippetUpdateRepositoryStorageWorker # rubocop:disable Scalability/IdempotentWorker - extend ::Gitlab::Utils::Override - include UpdateRepositoryStorageWorker - - private - - override :find_repository_storage_move - def find_repository_storage_move(repository_storage_move_id) - SnippetRepositoryStorageMove.find(repository_storage_move_id) - end - - override :find_container - def find_container(container_id) - Snippet.find(container_id) - end - - override :update_repository_storage - def update_repository_storage(repository_storage_move) - ::Snippets::UpdateRepositoryStorageService.new(repository_storage_move).execute - end +# This is a compatibility class to avoid calling a non-existent +# class from sidekiq during deployment. +# +# This class was moved to a namespace in https://gitlab.com/gitlab-org/gitlab/-/issues/299853. +# we cannot remove this class entirely because there can be jobs +# referencing it. +# +# We can get rid of this class in 14.0 +# https://gitlab.com/gitlab-org/gitlab/-/issues/322393 +class SnippetUpdateRepositoryStorageWorker < Snippets::UpdateRepositoryStorageWorker # rubocop:disable Scalability/IdempotentWorker + idempotent! + urgency :throttled end diff --git a/app/workers/snippets/schedule_bulk_repository_shard_moves_worker.rb b/app/workers/snippets/schedule_bulk_repository_shard_moves_worker.rb new file mode 100644 index 00000000000..ec3d9dbdf97 --- /dev/null +++ b/app/workers/snippets/schedule_bulk_repository_shard_moves_worker.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Snippets + class ScheduleBulkRepositoryShardMovesWorker + include ApplicationWorker + + idempotent! + feature_category :gitaly + urgency :throttled + + def perform(source_storage_name, destination_storage_name = nil) + Snippets::ScheduleBulkRepositoryShardMovesService.new.execute(source_storage_name, destination_storage_name) + end + end +end diff --git a/app/workers/snippets/update_repository_storage_worker.rb b/app/workers/snippets/update_repository_storage_worker.rb new file mode 100644 index 00000000000..83b655e9986 --- /dev/null +++ b/app/workers/snippets/update_repository_storage_worker.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module Snippets + class UpdateRepositoryStorageWorker # rubocop:disable Scalability/IdempotentWorker + extend ::Gitlab::Utils::Override + include ::UpdateRepositoryStorageWorker + + private + + override :find_repository_storage_move + def find_repository_storage_move(repository_storage_move_id) + Snippets::RepositoryStorageMove.find(repository_storage_move_id) + end + + override :find_container + def find_container(container_id) + Snippet.find(container_id) + end + + override :update_repository_storage + def update_repository_storage(repository_storage_move) + ::Snippets::UpdateRepositoryStorageService.new(repository_storage_move).execute + end + end +end diff --git a/app/workers/stuck_ci_jobs_worker.rb b/app/workers/stuck_ci_jobs_worker.rb index eca0a248a37..bd721df73c6 100644 --- a/app/workers/stuck_ci_jobs_worker.rb +++ b/app/workers/stuck_ci_jobs_worker.rb @@ -70,7 +70,7 @@ class StuckCiJobsWorker # rubocop:disable Scalability/IdempotentWorker def drop_build(type, build, status, timeout, reason) Gitlab::AppLogger.info "#{self.class}: Dropping #{type} build #{build.id} for runner #{build.runner_id} (status: #{status}, timeout: #{timeout}, reason: #{reason})" - Gitlab::OptimisticLocking.retry_lock(build, 3) do |b| + Gitlab::OptimisticLocking.retry_lock(build, 3, name: 'stuck_ci_jobs_worker_drop_build') do |b| b.drop(reason) end rescue => ex |