diff options
Diffstat (limited to 'app/workers')
28 files changed, 391 insertions, 208 deletions
diff --git a/app/workers/all_queues.yml b/app/workers/all_queues.yml index 4c4a314a1e6..59ab0a4d05b 100644 --- a/app/workers/all_queues.yml +++ b/app/workers/all_queues.yml @@ -251,6 +251,14 @@ :weight: 1 :idempotent: true :tags: [] +- :name: cronjob:namespaces_in_product_marketing_emails + :feature_category: :subgroups + :has_external_dependencies: + :urgency: :low + :resource_boundary: :unknown + :weight: 1 + :idempotent: + :tags: [] - :name: cronjob:namespaces_prune_aggregation_schedules :feature_category: :source_code_management :has_external_dependencies: @@ -259,6 +267,14 @@ :weight: 1 :idempotent: :tags: [] +- :name: cronjob:packages_composer_cache_cleanup + :feature_category: :package_registry + :has_external_dependencies: + :urgency: :low + :resource_boundary: :unknown + :weight: 1 + :idempotent: true + :tags: [] - :name: cronjob:pages_domain_removal_cron :feature_category: :pages :has_external_dependencies: @@ -459,6 +475,14 @@ :weight: 1 :idempotent: true :tags: [] +- :name: cronjob:user_status_cleanup_batch + :feature_category: :users + :has_external_dependencies: + :urgency: :low + :resource_boundary: :unknown + :weight: 1 + :idempotent: true + :tags: [] - :name: cronjob:users_create_statistics :feature_category: :users :has_external_dependencies: @@ -1076,24 +1100,23 @@ :idempotent: :tags: - :requires_disk_io -- :name: pipeline_background:ci_build_report_result +- :name: pipeline_background:ci_build_trace_chunk_flush :feature_category: :continuous_integration :has_external_dependencies: :urgency: :low :resource_boundary: :unknown :weight: 1 :idempotent: true - :tags: - - :requires_disk_io -- :name: pipeline_background:ci_build_trace_chunk_flush - :feature_category: :continuous_integration + :tags: [] +- :name: pipeline_background:ci_daily_build_group_report_results + :feature_category: :code_testing :has_external_dependencies: :urgency: :low :resource_boundary: :unknown :weight: 1 :idempotent: true :tags: [] -- :name: pipeline_background:ci_daily_build_group_report_results +- :name: pipeline_background:ci_pipeline_artifacts_coverage_report :feature_category: :code_testing :has_external_dependencies: :urgency: :low @@ -1101,7 +1124,7 @@ :weight: 1 :idempotent: true :tags: [] -- :name: pipeline_background:ci_pipeline_artifacts_coverage_report +- :name: pipeline_background:ci_pipeline_artifacts_create_quality_report :feature_category: :code_testing :has_external_dependencies: :urgency: :low @@ -1165,24 +1188,6 @@ :weight: 4 :idempotent: :tags: [] -- :name: pipeline_default:build_coverage - :feature_category: :continuous_integration - :has_external_dependencies: - :urgency: :low - :resource_boundary: :unknown - :weight: 3 - :idempotent: - :tags: - - :requires_disk_io -- :name: pipeline_default:build_trace_sections - :feature_category: :continuous_integration - :has_external_dependencies: - :urgency: :low - :resource_boundary: :unknown - :weight: 3 - :idempotent: - :tags: - - :requires_disk_io - :name: pipeline_default:ci_create_cross_project_pipeline :feature_category: :continuous_integration :has_external_dependencies: @@ -1999,6 +2004,14 @@ :weight: 1 :idempotent: true :tags: [] +- :name: projects_git_garbage_collect + :feature_category: :gitaly + :has_external_dependencies: + :urgency: :low + :resource_boundary: :unknown + :weight: 1 + :idempotent: + :tags: [] - :name: prometheus_create_default_alerts :feature_category: :incident_management :has_external_dependencies: @@ -2223,6 +2236,14 @@ :weight: 1 :idempotent: true :tags: [] +- :name: wikis_git_garbage_collect + :feature_category: :gitaly + :has_external_dependencies: + :urgency: :low + :resource_boundary: :unknown + :weight: 1 + :idempotent: + :tags: [] - :name: x509_certificate_revoke :feature_category: :source_code_management :has_external_dependencies: diff --git a/app/workers/authorized_projects_worker.rb b/app/workers/authorized_projects_worker.rb index f5132459131..6e07d6d0f71 100644 --- a/app/workers/authorized_projects_worker.rb +++ b/app/workers/authorized_projects_worker.rb @@ -25,7 +25,7 @@ class AuthorizedProjectsWorker def perform(user_id) user = User.find_by(id: user_id) - user&.refresh_authorized_projects + user&.refresh_authorized_projects(source: self.class.name) end # rubocop: enable CodeReuse/ActiveRecord end diff --git a/app/workers/build_coverage_worker.rb b/app/workers/build_coverage_worker.rb deleted file mode 100644 index d63d8549f09..00000000000 --- a/app/workers/build_coverage_worker.rb +++ /dev/null @@ -1,14 +0,0 @@ -# frozen_string_literal: true - -class BuildCoverageWorker # rubocop:disable Scalability/IdempotentWorker - include ApplicationWorker - include PipelineQueue - - tags :requires_disk_io - - # rubocop: disable CodeReuse/ActiveRecord - def perform(build_id) - Ci::Build.find_by(id: build_id)&.update_coverage - end - # rubocop: enable CodeReuse/ActiveRecord -end diff --git a/app/workers/build_finished_worker.rb b/app/workers/build_finished_worker.rb index d7a5fcf4f18..4d15bcd16f7 100644 --- a/app/workers/build_finished_worker.rb +++ b/app/workers/build_finished_worker.rb @@ -29,9 +29,9 @@ class BuildFinishedWorker # rubocop:disable Scalability/IdempotentWorker # @param [Ci::Build] build The build to process. def process_build(build) # We execute these in sync to reduce IO. - BuildTraceSectionsWorker.new.perform(build.id) - BuildCoverageWorker.new.perform(build.id) - Ci::BuildReportResultWorker.new.perform(build.id) + build.parse_trace_sections! + build.update_coverage + Ci::BuildReportResultService.new.execute(build) # We execute these async as these are independent operations. BuildHooksWorker.perform_async(build.id) diff --git a/app/workers/build_hooks_worker.rb b/app/workers/build_hooks_worker.rb index 9693d3eb57f..ce4aa7229aa 100644 --- a/app/workers/build_hooks_worker.rb +++ b/app/workers/build_hooks_worker.rb @@ -10,7 +10,8 @@ class BuildHooksWorker # rubocop:disable Scalability/IdempotentWorker # rubocop: disable CodeReuse/ActiveRecord def perform(build_id) - Ci::Build.find_by(id: build_id) + Ci::Build.includes({ runner: :tags }) + .find_by(id: build_id) .try(:execute_hooks) end # rubocop: enable CodeReuse/ActiveRecord diff --git a/app/workers/build_trace_sections_worker.rb b/app/workers/build_trace_sections_worker.rb deleted file mode 100644 index 59f019b827e..00000000000 --- a/app/workers/build_trace_sections_worker.rb +++ /dev/null @@ -1,14 +0,0 @@ -# frozen_string_literal: true - -class BuildTraceSectionsWorker # rubocop:disable Scalability/IdempotentWorker - include ApplicationWorker - include PipelineQueue - - tags :requires_disk_io - - # rubocop: disable CodeReuse/ActiveRecord - def perform(build_id) - Ci::Build.find_by(id: build_id)&.parse_trace_sections! - end - # rubocop: enable CodeReuse/ActiveRecord -end diff --git a/app/workers/bulk_import_worker.rb b/app/workers/bulk_import_worker.rb index 81099d4e5f7..e6bc54895a7 100644 --- a/app/workers/bulk_import_worker.rb +++ b/app/workers/bulk_import_worker.rb @@ -27,6 +27,10 @@ class BulkImportWorker # rubocop:disable Scalability/IdempotentWorker end re_enqueue + rescue => e + Gitlab::ErrorTracking.track_exception(e, bulk_import_id: @bulk_import&.id) + + @bulk_import&.fail_op end private diff --git a/app/workers/bulk_imports/entity_worker.rb b/app/workers/bulk_imports/entity_worker.rb index 9b29ad8f326..5b41ccbdea1 100644 --- a/app/workers/bulk_imports/entity_worker.rb +++ b/app/workers/bulk_imports/entity_worker.rb @@ -18,6 +18,16 @@ module BulkImports BulkImports::Importers::GroupImporter.new(entity).execute end + + rescue => e + extra = { + bulk_import_id: entity&.bulk_import&.id, + entity_id: entity&.id + } + + Gitlab::ErrorTracking.track_exception(e, extra) + + entity&.fail_op end end end diff --git a/app/workers/ci/build_report_result_worker.rb b/app/workers/ci/build_report_result_worker.rb deleted file mode 100644 index 01a45490541..00000000000 --- a/app/workers/ci/build_report_result_worker.rb +++ /dev/null @@ -1,18 +0,0 @@ -# frozen_string_literal: true - -module Ci - class BuildReportResultWorker - include ApplicationWorker - include PipelineBackgroundQueue - - tags :requires_disk_io - - idempotent! - - def perform(build_id) - Ci::Build.find_by_id(build_id).try do |build| - Ci::BuildReportResultService.new.execute(build) - end - end - end -end diff --git a/app/workers/ci/pipeline_artifacts/create_quality_report_worker.rb b/app/workers/ci/pipeline_artifacts/create_quality_report_worker.rb new file mode 100644 index 00000000000..810106e8d9c --- /dev/null +++ b/app/workers/ci/pipeline_artifacts/create_quality_report_worker.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +module Ci + module PipelineArtifacts + class CreateQualityReportWorker + include ApplicationWorker + + queue_namespace :pipeline_background + feature_category :code_testing + + idempotent! + + def perform(pipeline_id) + Ci::Pipeline.find_by_id(pipeline_id).try do |pipeline| + Ci::PipelineArtifacts::CreateCodeQualityMrDiffReportService.new.execute(pipeline) + end + end + end + end +end diff --git a/app/workers/concerns/git_garbage_collect_methods.rb b/app/workers/concerns/git_garbage_collect_methods.rb new file mode 100644 index 00000000000..17a80d1ddb3 --- /dev/null +++ b/app/workers/concerns/git_garbage_collect_methods.rb @@ -0,0 +1,135 @@ +# frozen_string_literal: true + +module GitGarbageCollectMethods + extend ActiveSupport::Concern + + included do + include ApplicationWorker + + sidekiq_options retry: false + feature_category :gitaly + loggable_arguments 1, 2, 3 + end + + # Timeout set to 24h + LEASE_TIMEOUT = 86400 + + def perform(resource_id, task = :gc, lease_key = nil, lease_uuid = nil) + resource = find_resource(resource_id) + lease_key ||= default_lease_key(task, resource) + active_uuid = get_lease_uuid(lease_key) + + if active_uuid + return unless active_uuid == lease_uuid + + renew_lease(lease_key, active_uuid) + else + lease_uuid = try_obtain_lease(lease_key) + + return unless lease_uuid + end + + task = task.to_sym + + before_gitaly_call(task, resource) + gitaly_call(task, resource) + + # Refresh the branch cache in case garbage collection caused a ref lookup to fail + flush_ref_caches(resource) if gc?(task) + + update_repository_statistics(resource) if task != :pack_refs + + # In case pack files are deleted, release libgit2 cache and open file + # descriptors ASAP instead of waiting for Ruby garbage collection + resource.cleanup + ensure + cancel_lease(lease_key, lease_uuid) if lease_key.present? && lease_uuid.present? + end + + private + + def default_lease_key(task, resource) + "git_gc:#{task}:#{resource.class.name.underscore.pluralize}:#{resource.id}" + end + + def find_resource(id) + raise NotImplementedError + end + + def gc?(task) + task == :gc || task == :prune + end + + def try_obtain_lease(key) + ::Gitlab::ExclusiveLease.new(key, timeout: LEASE_TIMEOUT).try_obtain + end + + def renew_lease(key, uuid) + ::Gitlab::ExclusiveLease.new(key, uuid: uuid, timeout: LEASE_TIMEOUT).renew + end + + def cancel_lease(key, uuid) + ::Gitlab::ExclusiveLease.cancel(key, uuid) + end + + def get_lease_uuid(key) + ::Gitlab::ExclusiveLease.get_uuid(key) + end + + def before_gitaly_call(task, resource) + # no-op + end + + def gitaly_call(task, resource) + repository = resource.repository.raw_repository + + client = get_gitaly_client(task, repository) + + case task + when :prune, :gc + client.garbage_collect(bitmaps_enabled?, prune: task == :prune) + when :full_repack + client.repack_full(bitmaps_enabled?) + when :incremental_repack + client.repack_incremental + when :pack_refs + client.pack_refs + end + rescue GRPC::NotFound => e + Gitlab::GitLogger.error("#{__method__} failed:\nRepository not found") + raise Gitlab::Git::Repository::NoRepository.new(e) + rescue GRPC::BadStatus => e + Gitlab::GitLogger.error("#{__method__} failed:\n#{e}") + raise Gitlab::Git::CommandError.new(e) + end + + def get_gitaly_client(task, repository) + if task == :pack_refs + Gitlab::GitalyClient::RefService + else + Gitlab::GitalyClient::RepositoryService + end.new(repository) + end + + def bitmaps_enabled? + Gitlab::CurrentSettings.housekeeping_bitmaps_enabled + end + + def flush_ref_caches(resource) + resource.repository.expire_branches_cache + resource.repository.branch_names + resource.repository.has_visible_content? + end + + def update_repository_statistics(resource) + resource.repository.expire_statistics_caches + + return if Gitlab::Database.read_only? # GitGarbageCollectWorker may be run on a Geo secondary + + update_db_repository_statistics(resource) + end + + def update_db_repository_statistics(resource) + # no-op + end +end diff --git a/app/workers/container_expiration_policies/cleanup_container_repository_worker.rb b/app/workers/container_expiration_policies/cleanup_container_repository_worker.rb index 7c86b194574..53220a7afed 100644 --- a/app/workers/container_expiration_policies/cleanup_container_repository_worker.rb +++ b/app/workers/container_expiration_policies/cleanup_container_repository_worker.rb @@ -18,6 +18,7 @@ module ContainerExpirationPolicies cleanup_tags_service_before_truncate_size cleanup_tags_service_after_truncate_size cleanup_tags_service_before_delete_size + cleanup_tags_service_deleted_size ].freeze def perform_work @@ -25,6 +26,7 @@ module ContainerExpirationPolicies return unless container_repository log_extra_metadata_on_done(:container_repository_id, container_repository.id) + log_extra_metadata_on_done(:project_id, project.id) unless allowed_to_run?(container_repository) container_repository.cleanup_unscheduled! @@ -78,7 +80,7 @@ module ContainerExpirationPolicies end def project - container_repository&.project + container_repository.project end def container_repository @@ -116,6 +118,7 @@ module ContainerExpirationPolicies after_truncate_size && before_truncate_size != after_truncate_size log_extra_metadata_on_done(:cleanup_tags_service_truncated, !!truncated) + log_extra_metadata_on_done(:running_jobs_count, running_jobs_count) end end end diff --git a/app/workers/git_garbage_collect_worker.rb b/app/workers/git_garbage_collect_worker.rb index e1dcb16bafb..a2aab23db7b 100644 --- a/app/workers/git_garbage_collect_worker.rb +++ b/app/workers/git_garbage_collect_worker.rb @@ -1,5 +1,11 @@ # frozen_string_literal: true +# According to our docs, we can only remove workers on major releases +# https://docs.gitlab.com/ee/development/sidekiq_style_guide.html#removing-workers. +# +# We need to still maintain this until 14.0 but with the current functionality. +# +# In https://gitlab.com/gitlab-org/gitlab/-/issues/299290 we track that removal. class GitGarbageCollectWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker @@ -7,117 +13,7 @@ class GitGarbageCollectWorker # rubocop:disable Scalability/IdempotentWorker feature_category :gitaly loggable_arguments 1, 2, 3 - # Timeout set to 24h - LEASE_TIMEOUT = 86400 - def perform(project_id, task = :gc, lease_key = nil, lease_uuid = nil) - lease_key ||= "git_gc:#{task}:#{project_id}" - project = Project.find(project_id) - active_uuid = get_lease_uuid(lease_key) - - if active_uuid - return unless active_uuid == lease_uuid - - renew_lease(lease_key, active_uuid) - else - lease_uuid = try_obtain_lease(lease_key) - - return unless lease_uuid - end - - task = task.to_sym - - if gc?(task) - ::Projects::GitDeduplicationService.new(project).execute - cleanup_orphan_lfs_file_references(project) - end - - gitaly_call(task, project) - - # Refresh the branch cache in case garbage collection caused a ref lookup to fail - flush_ref_caches(project) if gc?(task) - - update_repository_statistics(project) if task != :pack_refs - - # In case pack files are deleted, release libgit2 cache and open file - # descriptors ASAP instead of waiting for Ruby garbage collection - project.cleanup - ensure - cancel_lease(lease_key, lease_uuid) if lease_key.present? && lease_uuid.present? - end - - private - - def gc?(task) - task == :gc || task == :prune - end - - def try_obtain_lease(key) - ::Gitlab::ExclusiveLease.new(key, timeout: LEASE_TIMEOUT).try_obtain - end - - def renew_lease(key, uuid) - ::Gitlab::ExclusiveLease.new(key, uuid: uuid, timeout: LEASE_TIMEOUT).renew - end - - def cancel_lease(key, uuid) - ::Gitlab::ExclusiveLease.cancel(key, uuid) - end - - def get_lease_uuid(key) - ::Gitlab::ExclusiveLease.get_uuid(key) - end - - def gitaly_call(task, project) - repository = project.repository.raw_repository - - client = if task == :pack_refs - Gitlab::GitalyClient::RefService.new(repository) - else - Gitlab::GitalyClient::RepositoryService.new(repository) - end - - case task - when :prune, :gc - client.garbage_collect(bitmaps_enabled?, prune: task == :prune) - when :full_repack - client.repack_full(bitmaps_enabled?) - when :incremental_repack - client.repack_incremental - when :pack_refs - client.pack_refs - end - rescue GRPC::NotFound => e - Gitlab::GitLogger.error("#{__method__} failed:\nRepository not found") - raise Gitlab::Git::Repository::NoRepository.new(e) - rescue GRPC::BadStatus => e - Gitlab::GitLogger.error("#{__method__} failed:\n#{e}") - raise Gitlab::Git::CommandError.new(e) - end - - def cleanup_orphan_lfs_file_references(project) - return if Gitlab::Database.read_only? # GitGarbageCollectWorker may be run on a Geo secondary - - ::Gitlab::Cleanup::OrphanLfsFileReferences.new(project, dry_run: false, logger: logger).run! - rescue => err - Gitlab::GitLogger.warn(message: "Cleaning up orphan LFS objects files failed", error: err.message) - Gitlab::ErrorTracking.track_and_raise_for_dev_exception(err) - end - - def flush_ref_caches(project) - project.repository.expire_branches_cache - project.repository.branch_names - project.repository.has_visible_content? - end - - def update_repository_statistics(project) - project.repository.expire_statistics_caches - return if Gitlab::Database.read_only? # GitGarbageCollectWorker may be run on a Geo secondary - - Projects::UpdateStatisticsService.new(project, nil, statistics: [:repository_size, :lfs_objects_size]).execute - end - - def bitmaps_enabled? - Gitlab::CurrentSettings.housekeeping_bitmaps_enabled + ::Projects::GitGarbageCollectWorker.new.perform(project_id, task, lease_key, lease_uuid) end end diff --git a/app/workers/issuable_export_csv_worker.rb b/app/workers/issuable_export_csv_worker.rb index 33452b14edb..eb96a78497c 100644 --- a/app/workers/issuable_export_csv_worker.rb +++ b/app/workers/issuable_export_csv_worker.rb @@ -10,29 +10,21 @@ class IssuableExportCsvWorker # rubocop:disable Scalability/IdempotentWorker def perform(type, current_user_id, project_id, params) user = User.find(current_user_id) project = Project.find(project_id) - finder_params = map_params(params, project_id) - export_service(type.to_sym, user, project, finder_params).email(user) + export_service(type, user, project, params).email(user) rescue ActiveRecord::RecordNotFound => error logger.error("Failed to export CSV (current_user_id:#{current_user_id}, project_id:#{project_id}): #{error.message}") end private - def map_params(params, project_id) - params - .symbolize_keys - .except(:sort) - .merge(project_id: project_id) - end - def export_service(type, user, project, params) - issuable_class = service_classes_for(type) - issuables = issuable_class[:finder].new(user, params).execute - issuable_class[:service].new(issuables, project) + issuable_classes = issuable_classes_for(type.to_sym) + issuables = issuable_classes[:finder].new(user, parse_params(params, project.id)).execute + issuable_classes[:service].new(issuables, project) end - def service_classes_for(type) + def issuable_classes_for(type) case type when :issue { finder: IssuesFinder, service: Issues::ExportCsvService } @@ -43,6 +35,13 @@ class IssuableExportCsvWorker # rubocop:disable Scalability/IdempotentWorker end end + def parse_params(params, project_id) + params + .symbolize_keys + .except(:sort) + .merge(project_id: project_id) + end + def type_error_message(type) "Type parameter must be :issue or :merge_request, it was #{type}" end diff --git a/app/workers/jira_connect/sync_builds_worker.rb b/app/workers/jira_connect/sync_builds_worker.rb index c1c749f6041..9cb5d5d247d 100644 --- a/app/workers/jira_connect/sync_builds_worker.rb +++ b/app/workers/jira_connect/sync_builds_worker.rb @@ -14,7 +14,6 @@ module JiraConnect pipeline = Ci::Pipeline.find_by_id(pipeline_id) return unless pipeline - return unless Feature.enabled?(:jira_sync_builds, pipeline.project) ::JiraConnect::SyncService .new(pipeline.project) diff --git a/app/workers/jira_connect/sync_deployments_worker.rb b/app/workers/jira_connect/sync_deployments_worker.rb index 0f261e29464..7272d35f4cb 100644 --- a/app/workers/jira_connect/sync_deployments_worker.rb +++ b/app/workers/jira_connect/sync_deployments_worker.rb @@ -14,7 +14,6 @@ module JiraConnect deployment = Deployment.find_by_id(deployment_id) return unless deployment - return unless Feature.enabled?(:jira_sync_deployments, deployment.project) ::JiraConnect::SyncService .new(deployment.project) diff --git a/app/workers/jira_connect/sync_feature_flags_worker.rb b/app/workers/jira_connect/sync_feature_flags_worker.rb index 7e98d0eada7..496b9f1626d 100644 --- a/app/workers/jira_connect/sync_feature_flags_worker.rb +++ b/app/workers/jira_connect/sync_feature_flags_worker.rb @@ -14,7 +14,6 @@ module JiraConnect feature_flag = ::Operations::FeatureFlag.find_by_id(feature_flag_id) return unless feature_flag - return unless Feature.enabled?(:jira_sync_feature_flags, feature_flag.project) ::JiraConnect::SyncService .new(feature_flag.project) diff --git a/app/workers/merge_request_cleanup_refs_worker.rb b/app/workers/merge_request_cleanup_refs_worker.rb index 6b991a2253f..fbd62ac0a91 100644 --- a/app/workers/merge_request_cleanup_refs_worker.rb +++ b/app/workers/merge_request_cleanup_refs_worker.rb @@ -7,6 +7,8 @@ class MergeRequestCleanupRefsWorker idempotent! def perform(merge_request_id) + return unless Feature.enabled?(:merge_request_refs_cleanup, default_enabled: false) + merge_request = MergeRequest.find_by_id(merge_request_id) unless merge_request diff --git a/app/workers/namespaces/in_product_marketing_emails_worker.rb b/app/workers/namespaces/in_product_marketing_emails_worker.rb new file mode 100644 index 00000000000..66d140928a7 --- /dev/null +++ b/app/workers/namespaces/in_product_marketing_emails_worker.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module Namespaces + class InProductMarketingEmailsWorker # rubocop:disable Scalability/IdempotentWorker + include ApplicationWorker + include CronjobQueue # rubocop:disable Scalability/CronWorkerContext + + feature_category :subgroups + urgency :low + + def perform + return unless Gitlab::Experimentation.active?(:in_product_marketing_emails) + + Namespaces::InProductMarketingEmailsService.send_for_all_tracks_and_intervals + end + end +end diff --git a/app/workers/packages/composer/cache_cleanup_worker.rb b/app/workers/packages/composer/cache_cleanup_worker.rb new file mode 100644 index 00000000000..638e50e18c4 --- /dev/null +++ b/app/workers/packages/composer/cache_cleanup_worker.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +module Packages + module Composer + class CacheCleanupWorker + include ApplicationWorker + include CronjobQueue # rubocop:disable Scalability/CronWorkerContext + + feature_category :package_registry + + idempotent! + + def perform + ::Packages::Composer::CacheFile.without_namespace.find_in_batches do |cache_files| + cache_files.each(&:destroy) + rescue ActiveRecord::RecordNotFound + # ignore. likely due to object already being deleted. + end + + ::Packages::Composer::CacheFile.expired.find_in_batches do |cache_files| + cache_files.each(&:destroy) + rescue ActiveRecord::RecordNotFound + # ignore. likely due to object already being deleted. + end + rescue => e + Gitlab::ErrorTracking.log_exception(e) + end + end + end +end diff --git a/app/workers/pages_remove_worker.rb b/app/workers/pages_remove_worker.rb index b83168fd7bd..67ea18545a7 100644 --- a/app/workers/pages_remove_worker.rb +++ b/app/workers/pages_remove_worker.rb @@ -1,5 +1,6 @@ # frozen_string_literal: true +# TODO: remove this worker https://gitlab.com/gitlab-org/gitlab/-/issues/320775 class PagesRemoveWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker @@ -11,7 +12,6 @@ class PagesRemoveWorker # rubocop:disable Scalability/IdempotentWorker project = Project.find_by_id(project_id) return unless project - project.remove_pages - project.pages_domains.delete_all + project.legacy_remove_pages end end diff --git a/app/workers/pages_transfer_worker.rb b/app/workers/pages_transfer_worker.rb index f78564cc69d..5d395c9e38a 100644 --- a/app/workers/pages_transfer_worker.rb +++ b/app/workers/pages_transfer_worker.rb @@ -9,7 +9,7 @@ class PagesTransferWorker # rubocop:disable Scalability/IdempotentWorker loggable_arguments 0, 1 def perform(method, args) - return unless Gitlab::PagesTransfer::Async::METHODS.include?(method) + return unless Gitlab::PagesTransfer::METHODS.include?(method) result = Gitlab::PagesTransfer.new.public_send(method, *args) # rubocop:disable GitlabSecurity/PublicSend diff --git a/app/workers/pipeline_hooks_worker.rb b/app/workers/pipeline_hooks_worker.rb index 85ecdd02fb5..b8dd4768cfb 100644 --- a/app/workers/pipeline_hooks_worker.rb +++ b/app/workers/pipeline_hooks_worker.rb @@ -10,7 +10,8 @@ class PipelineHooksWorker # rubocop:disable Scalability/IdempotentWorker # rubocop: disable CodeReuse/ActiveRecord def perform(pipeline_id) - Ci::Pipeline.find_by(id: pipeline_id) + Ci::Pipeline.includes({ builds: { runner: :tags } }) + .find_by(id: pipeline_id) .try(:execute_hooks) end # rubocop: enable CodeReuse/ActiveRecord diff --git a/app/workers/post_receive.rb b/app/workers/post_receive.rb index 9fe7dd31e68..ac55f883fc5 100644 --- a/app/workers/post_receive.rb +++ b/app/workers/post_receive.rb @@ -2,6 +2,7 @@ class PostReceive # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + include Gitlab::Experiment::Dsl feature_category :source_code_management urgency :high @@ -121,6 +122,7 @@ class PostReceive # rubocop:disable Scalability/IdempotentWorker end def after_project_changes_hooks(project, user, refs, changes) + experiment(:new_project_readme, actor: user).track_initial_writes(project) repository_update_hook_data = Gitlab::DataBuilder::Repository.update(project, user, changes, refs) SystemHooksService.new.execute_hooks(repository_update_hook_data, :repository_update_hooks) Gitlab::UsageDataCounters::SourceCodeCounter.count(:pushes) diff --git a/app/workers/projects/git_garbage_collect_worker.rb b/app/workers/projects/git_garbage_collect_worker.rb new file mode 100644 index 00000000000..4f908529b34 --- /dev/null +++ b/app/workers/projects/git_garbage_collect_worker.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +module Projects + class GitGarbageCollectWorker # rubocop:disable Scalability/IdempotentWorker + extend ::Gitlab::Utils::Override + include GitGarbageCollectMethods + + private + + override :find_resource + def find_resource(id) + Project.find(id) + end + + override :before_gitaly_call + def before_gitaly_call(task, resource) + return unless gc?(task) + + ::Projects::GitDeduplicationService.new(resource).execute + cleanup_orphan_lfs_file_references(resource) + end + + def cleanup_orphan_lfs_file_references(resource) + return if Gitlab::Database.read_only? # GitGarbageCollectWorker may be run on a Geo secondary + + ::Gitlab::Cleanup::OrphanLfsFileReferences.new(resource, dry_run: false, logger: logger).run! + rescue => err + Gitlab::GitLogger.warn(message: "Cleaning up orphan LFS objects files failed", error: err.message) + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(err) + end + + override :update_db_repository_statistics + def update_db_repository_statistics(resource) + Projects::UpdateStatisticsService.new(resource, nil, statistics: [:repository_size, :lfs_objects_size]).execute + end + end +end diff --git a/app/workers/schedule_merge_request_cleanup_refs_worker.rb b/app/workers/schedule_merge_request_cleanup_refs_worker.rb index 59b8993f78f..967032f99e5 100644 --- a/app/workers/schedule_merge_request_cleanup_refs_worker.rb +++ b/app/workers/schedule_merge_request_cleanup_refs_worker.rb @@ -16,6 +16,7 @@ class ScheduleMergeRequestCleanupRefsWorker def perform return if Gitlab::Database.read_only? + return unless Feature.enabled?(:merge_request_refs_cleanup, default_enabled: false) ids = MergeRequest::CleanupSchedule.scheduled_merge_request_ids(LIMIT).map { |id| [id] } diff --git a/app/workers/user_status_cleanup/batch_worker.rb b/app/workers/user_status_cleanup/batch_worker.rb new file mode 100644 index 00000000000..0c1087cc4d2 --- /dev/null +++ b/app/workers/user_status_cleanup/batch_worker.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +module UserStatusCleanup + # This worker will run every minute to look for user status records to clean up. + class BatchWorker + include ApplicationWorker + # rubocop:disable Scalability/CronWorkerContext + include CronjobQueue + # rubocop:enable Scalability/CronWorkerContext + + feature_category :users + + idempotent! + + # Avoid running too many UPDATE queries at once + MAX_RUNTIME = 30.seconds + + def perform + return unless UserStatus.scheduled_for_cleanup.exists? + + start_time = Process.clock_gettime(Process::CLOCK_MONOTONIC) + + loop do + result = Users::BatchStatusCleanerService.execute + break if result[:deleted_rows] < Users::BatchStatusCleanerService::BATCH_SIZE + + current_time = Process.clock_gettime(Process::CLOCK_MONOTONIC) + + break if (current_time - start_time) > MAX_RUNTIME + end + end + end +end diff --git a/app/workers/wikis/git_garbage_collect_worker.rb b/app/workers/wikis/git_garbage_collect_worker.rb new file mode 100644 index 00000000000..1b455c50618 --- /dev/null +++ b/app/workers/wikis/git_garbage_collect_worker.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +module Wikis + class GitGarbageCollectWorker # rubocop:disable Scalability/IdempotentWorker + extend ::Gitlab::Utils::Override + include GitGarbageCollectMethods + + private + + override :find_resource + def find_resource(id) + Project.find(id).wiki + end + + override :update_db_repository_statistics + def update_db_repository_statistics(resource) + Projects::UpdateStatisticsService.new(resource.container, nil, statistics: [:wiki_size]).execute + end + end +end |