diff options
Diffstat (limited to 'app/workers')
79 files changed, 715 insertions, 172 deletions
diff --git a/app/workers/all_queues.yml b/app/workers/all_queues.yml index 1f9a53d64d9..0699be0f4cb 100644 --- a/app/workers/all_queues.yml +++ b/app/workers/all_queues.yml @@ -10,6 +10,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: true + :tags: [] - :name: authorized_project_update:authorized_project_update_user_refresh_with_low_urgency :feature_category: :authentication_and_authorization :has_external_dependencies: @@ -17,6 +18,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: true + :tags: [] - :name: auto_devops:auto_devops_disable :feature_category: :auto_devops :has_external_dependencies: @@ -24,6 +26,7 @@ :resource_boundary: :unknown :weight: 2 :idempotent: + :tags: [] - :name: auto_merge:auto_merge_process :feature_category: :continuous_delivery :has_external_dependencies: @@ -31,6 +34,7 @@ :resource_boundary: :cpu :weight: 3 :idempotent: + :tags: [] - :name: chaos:chaos_cpu_spin :feature_category: :not_owned :has_external_dependencies: @@ -38,6 +42,7 @@ :resource_boundary: :unknown :weight: 2 :idempotent: + :tags: [] - :name: chaos:chaos_db_spin :feature_category: :not_owned :has_external_dependencies: @@ -45,6 +50,7 @@ :resource_boundary: :unknown :weight: 2 :idempotent: + :tags: [] - :name: chaos:chaos_kill :feature_category: :not_owned :has_external_dependencies: @@ -52,6 +58,7 @@ :resource_boundary: :unknown :weight: 2 :idempotent: + :tags: [] - :name: chaos:chaos_leak_mem :feature_category: :not_owned :has_external_dependencies: @@ -59,6 +66,7 @@ :resource_boundary: :unknown :weight: 2 :idempotent: + :tags: [] - :name: chaos:chaos_sleep :feature_category: :not_owned :has_external_dependencies: @@ -66,6 +74,7 @@ :resource_boundary: :unknown :weight: 2 :idempotent: + :tags: [] - :name: container_repository:cleanup_container_repository :feature_category: :container_registry :has_external_dependencies: @@ -73,6 +82,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: + :tags: [] - :name: container_repository:delete_container_repository :feature_category: :container_registry :has_external_dependencies: @@ -80,6 +90,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: + :tags: [] - :name: cronjob:admin_email :feature_category: :source_code_management :has_external_dependencies: @@ -87,6 +98,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: + :tags: [] - :name: cronjob:ci_archive_traces_cron :feature_category: :continuous_integration :has_external_dependencies: @@ -94,6 +106,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: + :tags: [] - :name: cronjob:container_expiration_policy :feature_category: :container_registry :has_external_dependencies: @@ -101,6 +114,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: + :tags: [] - :name: cronjob:environments_auto_stop_cron :feature_category: :continuous_delivery :has_external_dependencies: @@ -108,6 +122,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: + :tags: [] - :name: cronjob:expire_build_artifacts :feature_category: :continuous_integration :has_external_dependencies: @@ -115,6 +130,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: + :tags: [] - :name: cronjob:gitlab_usage_ping :feature_category: :collection :has_external_dependencies: @@ -122,6 +138,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: + :tags: [] - :name: cronjob:import_export_project_cleanup :feature_category: :importers :has_external_dependencies: @@ -129,6 +146,15 @@ :resource_boundary: :unknown :weight: 1 :idempotent: + :tags: [] +- :name: cronjob:import_stuck_project_import_jobs + :feature_category: :importers + :has_external_dependencies: + :urgency: :low + :resource_boundary: :cpu + :weight: 1 + :idempotent: + :tags: [] - :name: cronjob:issue_due_scheduler :feature_category: :issue_tracking :has_external_dependencies: @@ -136,6 +162,23 @@ :resource_boundary: :unknown :weight: 1 :idempotent: + :tags: [] +- :name: cronjob:jira_import_stuck_jira_import_jobs + :feature_category: :importers + :has_external_dependencies: + :urgency: :low + :resource_boundary: :cpu + :weight: 1 + :idempotent: + :tags: [] +- :name: cronjob:metrics_dashboard_schedule_annotations_prune + :feature_category: :metrics + :has_external_dependencies: + :urgency: :low + :resource_boundary: :unknown + :weight: 1 + :idempotent: true + :tags: [] - :name: cronjob:namespaces_prune_aggregation_schedules :feature_category: :source_code_management :has_external_dependencies: @@ -143,6 +186,7 @@ :resource_boundary: :cpu :weight: 1 :idempotent: + :tags: [] - :name: cronjob:pages_domain_removal_cron :feature_category: :pages :has_external_dependencies: @@ -150,6 +194,7 @@ :resource_boundary: :cpu :weight: 1 :idempotent: + :tags: [] - :name: cronjob:pages_domain_ssl_renewal_cron :feature_category: :pages :has_external_dependencies: @@ -157,6 +202,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: + :tags: [] - :name: cronjob:pages_domain_verification_cron :feature_category: :pages :has_external_dependencies: @@ -164,6 +210,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: + :tags: [] - :name: cronjob:personal_access_tokens_expiring :feature_category: :authentication_and_authorization :has_external_dependencies: @@ -171,6 +218,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: + :tags: [] - :name: cronjob:pipeline_schedule :feature_category: :continuous_integration :has_external_dependencies: @@ -178,6 +226,7 @@ :resource_boundary: :cpu :weight: 1 :idempotent: + :tags: [] - :name: cronjob:prune_old_events :feature_category: :users :has_external_dependencies: @@ -185,6 +234,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: + :tags: [] - :name: cronjob:prune_web_hook_logs :feature_category: :integrations :has_external_dependencies: @@ -192,6 +242,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: + :tags: [] - :name: cronjob:remove_expired_group_links :feature_category: :authentication_and_authorization :has_external_dependencies: @@ -199,6 +250,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: + :tags: [] - :name: cronjob:remove_expired_members :feature_category: :authentication_and_authorization :has_external_dependencies: @@ -206,6 +258,7 @@ :resource_boundary: :cpu :weight: 1 :idempotent: + :tags: [] - :name: cronjob:remove_unreferenced_lfs_objects :feature_category: :git_lfs :has_external_dependencies: @@ -213,6 +266,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: + :tags: [] - :name: cronjob:repository_archive_cache :feature_category: :source_code_management :has_external_dependencies: @@ -220,6 +274,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: + :tags: [] - :name: cronjob:repository_check_dispatch :feature_category: :source_code_management :has_external_dependencies: @@ -227,6 +282,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: + :tags: [] - :name: cronjob:requests_profiles :feature_category: :source_code_management :has_external_dependencies: @@ -234,6 +290,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: + :tags: [] - :name: cronjob:schedule_migrate_external_diffs :feature_category: :source_code_management :has_external_dependencies: @@ -241,6 +298,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: + :tags: [] - :name: cronjob:stuck_ci_jobs :feature_category: :continuous_integration :has_external_dependencies: @@ -248,6 +306,7 @@ :resource_boundary: :cpu :weight: 1 :idempotent: + :tags: [] - :name: cronjob:stuck_export_jobs :feature_category: :importers :has_external_dependencies: @@ -255,6 +314,7 @@ :resource_boundary: :cpu :weight: 1 :idempotent: + :tags: [] - :name: cronjob:stuck_import_jobs :feature_category: :importers :has_external_dependencies: @@ -262,6 +322,7 @@ :resource_boundary: :cpu :weight: 1 :idempotent: + :tags: [] - :name: cronjob:stuck_merge_jobs :feature_category: :source_code_management :has_external_dependencies: @@ -269,6 +330,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: + :tags: [] - :name: cronjob:trending_projects :feature_category: :source_code_management :has_external_dependencies: @@ -276,6 +338,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: + :tags: [] - :name: cronjob:users_create_statistics :feature_category: :users :has_external_dependencies: @@ -283,6 +346,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: + :tags: [] - :name: cronjob:x509_issuer_crl_check :feature_category: :source_code_management :has_external_dependencies: true @@ -290,6 +354,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: true + :tags: [] - :name: deployment:deployments_finished :feature_category: :continuous_delivery :has_external_dependencies: @@ -297,6 +362,7 @@ :resource_boundary: :cpu :weight: 3 :idempotent: + :tags: [] - :name: deployment:deployments_forward_deployment :feature_category: :continuous_delivery :has_external_dependencies: @@ -304,6 +370,7 @@ :resource_boundary: :unknown :weight: 3 :idempotent: + :tags: [] - :name: deployment:deployments_success :feature_category: :continuous_delivery :has_external_dependencies: @@ -311,6 +378,7 @@ :resource_boundary: :cpu :weight: 3 :idempotent: + :tags: [] - :name: gcp_cluster:cluster_configure_istio :feature_category: :kubernetes_management :has_external_dependencies: true @@ -318,6 +386,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: + :tags: [] - :name: gcp_cluster:cluster_install_app :feature_category: :kubernetes_management :has_external_dependencies: true @@ -325,6 +394,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: + :tags: [] - :name: gcp_cluster:cluster_patch_app :feature_category: :kubernetes_management :has_external_dependencies: true @@ -332,6 +402,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: + :tags: [] - :name: gcp_cluster:cluster_provision :feature_category: :kubernetes_management :has_external_dependencies: true @@ -339,6 +410,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: + :tags: [] - :name: gcp_cluster:cluster_update_app :feature_category: :kubernetes_management :has_external_dependencies: @@ -346,6 +418,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: + :tags: [] - :name: gcp_cluster:cluster_upgrade_app :feature_category: :kubernetes_management :has_external_dependencies: true @@ -353,6 +426,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: + :tags: [] - :name: gcp_cluster:cluster_wait_for_app_installation :feature_category: :kubernetes_management :has_external_dependencies: true @@ -360,6 +434,7 @@ :resource_boundary: :cpu :weight: 1 :idempotent: + :tags: [] - :name: gcp_cluster:cluster_wait_for_app_update :feature_category: :kubernetes_management :has_external_dependencies: @@ -367,6 +442,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: + :tags: [] - :name: gcp_cluster:cluster_wait_for_ingress_ip_address :feature_category: :kubernetes_management :has_external_dependencies: true @@ -374,6 +450,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: + :tags: [] - :name: gcp_cluster:clusters_applications_activate_service :feature_category: :kubernetes_management :has_external_dependencies: @@ -381,6 +458,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: + :tags: [] - :name: gcp_cluster:clusters_applications_deactivate_service :feature_category: :kubernetes_management :has_external_dependencies: @@ -388,6 +466,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: + :tags: [] - :name: gcp_cluster:clusters_applications_uninstall :feature_category: :kubernetes_management :has_external_dependencies: true @@ -395,6 +474,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: + :tags: [] - :name: gcp_cluster:clusters_applications_wait_for_uninstall_app :feature_category: :kubernetes_management :has_external_dependencies: true @@ -402,6 +482,7 @@ :resource_boundary: :cpu :weight: 1 :idempotent: + :tags: [] - :name: gcp_cluster:clusters_cleanup_app :feature_category: :kubernetes_management :has_external_dependencies: true @@ -409,6 +490,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: + :tags: [] - :name: gcp_cluster:clusters_cleanup_project_namespace :feature_category: :kubernetes_management :has_external_dependencies: true @@ -416,6 +498,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: + :tags: [] - :name: gcp_cluster:clusters_cleanup_service_account :feature_category: :kubernetes_management :has_external_dependencies: true @@ -423,6 +506,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: + :tags: [] - :name: gcp_cluster:wait_for_cluster_creation :feature_category: :kubernetes_management :has_external_dependencies: true @@ -430,6 +514,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: + :tags: [] - :name: github_importer:github_import_import_diff_note :feature_category: :importers :has_external_dependencies: true @@ -437,6 +522,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: + :tags: [] - :name: github_importer:github_import_import_issue :feature_category: :importers :has_external_dependencies: true @@ -444,6 +530,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: + :tags: [] - :name: github_importer:github_import_import_lfs_object :feature_category: :importers :has_external_dependencies: true @@ -451,6 +538,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: + :tags: [] - :name: github_importer:github_import_import_note :feature_category: :importers :has_external_dependencies: true @@ -458,6 +546,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: + :tags: [] - :name: github_importer:github_import_import_pull_request :feature_category: :importers :has_external_dependencies: true @@ -465,6 +554,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: + :tags: [] - :name: github_importer:github_import_refresh_import_jid :feature_category: :importers :has_external_dependencies: @@ -472,6 +562,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: + :tags: [] - :name: github_importer:github_import_stage_finish_import :feature_category: :importers :has_external_dependencies: @@ -479,6 +570,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: + :tags: [] - :name: github_importer:github_import_stage_import_base_data :feature_category: :importers :has_external_dependencies: @@ -486,6 +578,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: + :tags: [] - :name: github_importer:github_import_stage_import_issues_and_diff_notes :feature_category: :importers :has_external_dependencies: @@ -493,6 +586,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: + :tags: [] - :name: github_importer:github_import_stage_import_lfs_objects :feature_category: :importers :has_external_dependencies: @@ -500,6 +594,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: + :tags: [] - :name: github_importer:github_import_stage_import_notes :feature_category: :importers :has_external_dependencies: @@ -507,6 +602,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: + :tags: [] - :name: github_importer:github_import_stage_import_pull_requests :feature_category: :importers :has_external_dependencies: @@ -514,6 +610,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: + :tags: [] - :name: github_importer:github_import_stage_import_repository :feature_category: :importers :has_external_dependencies: @@ -521,6 +618,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: + :tags: [] - :name: hashed_storage:hashed_storage_migrator :feature_category: :source_code_management :has_external_dependencies: @@ -528,6 +626,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: + :tags: [] - :name: hashed_storage:hashed_storage_project_migrate :feature_category: :source_code_management :has_external_dependencies: @@ -535,6 +634,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: + :tags: [] - :name: hashed_storage:hashed_storage_project_rollback :feature_category: :source_code_management :has_external_dependencies: @@ -542,6 +642,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: + :tags: [] - :name: hashed_storage:hashed_storage_rollbacker :feature_category: :source_code_management :has_external_dependencies: @@ -549,6 +650,15 @@ :resource_boundary: :unknown :weight: 1 :idempotent: + :tags: [] +- :name: incident_management:clusters_applications_check_prometheus_health + :feature_category: :incident_management + :has_external_dependencies: true + :urgency: :low + :resource_boundary: :unknown + :weight: 2 + :idempotent: true + :tags: [] - :name: incident_management:incident_management_process_alert :feature_category: :incident_management :has_external_dependencies: @@ -556,6 +666,7 @@ :resource_boundary: :unknown :weight: 2 :idempotent: + :tags: [] - :name: incident_management:incident_management_process_prometheus_alert :feature_category: :incident_management :has_external_dependencies: @@ -563,6 +674,7 @@ :resource_boundary: :cpu :weight: 2 :idempotent: + :tags: [] - :name: jira_importer:jira_import_advance_stage :feature_category: :importers :has_external_dependencies: @@ -570,6 +682,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: + :tags: [] - :name: jira_importer:jira_import_import_issue :feature_category: :importers :has_external_dependencies: @@ -577,6 +690,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: + :tags: [] - :name: jira_importer:jira_import_stage_finish_import :feature_category: :importers :has_external_dependencies: @@ -584,6 +698,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: + :tags: [] - :name: jira_importer:jira_import_stage_import_attachments :feature_category: :importers :has_external_dependencies: @@ -591,6 +706,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: + :tags: [] - :name: jira_importer:jira_import_stage_import_issues :feature_category: :importers :has_external_dependencies: @@ -598,6 +714,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: + :tags: [] - :name: jira_importer:jira_import_stage_import_labels :feature_category: :importers :has_external_dependencies: @@ -605,6 +722,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: + :tags: [] - :name: jira_importer:jira_import_stage_import_notes :feature_category: :importers :has_external_dependencies: @@ -612,6 +730,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: + :tags: [] - :name: jira_importer:jira_import_stage_start_import :feature_category: :importers :has_external_dependencies: @@ -619,6 +738,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: + :tags: [] - :name: mail_scheduler:mail_scheduler_issue_due :feature_category: :issue_tracking :has_external_dependencies: @@ -626,6 +746,7 @@ :resource_boundary: :unknown :weight: 2 :idempotent: + :tags: [] - :name: mail_scheduler:mail_scheduler_notification_service :feature_category: :issue_tracking :has_external_dependencies: @@ -633,6 +754,7 @@ :resource_boundary: :cpu :weight: 2 :idempotent: + :tags: [] - :name: notifications:new_release :feature_category: :release_orchestration :has_external_dependencies: @@ -640,6 +762,7 @@ :resource_boundary: :unknown :weight: 2 :idempotent: + :tags: [] - :name: object_pool:object_pool_create :feature_category: :gitaly :has_external_dependencies: @@ -647,6 +770,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: + :tags: [] - :name: object_pool:object_pool_destroy :feature_category: :gitaly :has_external_dependencies: @@ -654,6 +778,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: + :tags: [] - :name: object_pool:object_pool_join :feature_category: :gitaly :has_external_dependencies: @@ -661,6 +786,7 @@ :resource_boundary: :cpu :weight: 1 :idempotent: + :tags: [] - :name: object_pool:object_pool_schedule_join :feature_category: :gitaly :has_external_dependencies: @@ -668,6 +794,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: + :tags: [] - :name: object_storage:object_storage_background_move :feature_category: :not_owned :has_external_dependencies: @@ -675,6 +802,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: + :tags: [] - :name: object_storage:object_storage_migrate_uploads :feature_category: :not_owned :has_external_dependencies: @@ -682,6 +810,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: + :tags: [] - :name: pipeline_background:archive_trace :feature_category: :continuous_integration :has_external_dependencies: @@ -689,6 +818,15 @@ :resource_boundary: :unknown :weight: 1 :idempotent: + :tags: [] +- :name: pipeline_background:ci_build_report_result + :feature_category: :continuous_integration + :has_external_dependencies: + :urgency: :low + :resource_boundary: :unknown + :weight: 1 + :idempotent: true + :tags: [] - :name: pipeline_background:ci_build_trace_chunk_flush :feature_category: :continuous_integration :has_external_dependencies: @@ -696,6 +834,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: + :tags: [] - :name: pipeline_background:ci_daily_build_group_report_results :feature_category: :continuous_integration :has_external_dependencies: @@ -703,6 +842,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: true + :tags: [] - :name: pipeline_cache:expire_job_cache :feature_category: :continuous_integration :has_external_dependencies: @@ -710,6 +850,7 @@ :resource_boundary: :unknown :weight: 3 :idempotent: true + :tags: [] - :name: pipeline_cache:expire_pipeline_cache :feature_category: :continuous_integration :has_external_dependencies: @@ -717,6 +858,7 @@ :resource_boundary: :cpu :weight: 3 :idempotent: true + :tags: [] - :name: pipeline_creation:create_pipeline :feature_category: :continuous_integration :has_external_dependencies: @@ -724,6 +866,7 @@ :resource_boundary: :cpu :weight: 4 :idempotent: + :tags: [] - :name: pipeline_creation:run_pipeline_schedule :feature_category: :continuous_integration :has_external_dependencies: @@ -731,6 +874,7 @@ :resource_boundary: :unknown :weight: 4 :idempotent: + :tags: [] - :name: pipeline_default:build_coverage :feature_category: :continuous_integration :has_external_dependencies: @@ -738,6 +882,7 @@ :resource_boundary: :unknown :weight: 3 :idempotent: + :tags: [] - :name: pipeline_default:build_trace_sections :feature_category: :continuous_integration :has_external_dependencies: @@ -745,6 +890,7 @@ :resource_boundary: :unknown :weight: 3 :idempotent: + :tags: [] - :name: pipeline_default:ci_create_cross_project_pipeline :feature_category: :continuous_integration :has_external_dependencies: @@ -752,6 +898,7 @@ :resource_boundary: :cpu :weight: 3 :idempotent: + :tags: [] - :name: pipeline_default:ci_pipeline_bridge_status :feature_category: :continuous_integration :has_external_dependencies: @@ -759,6 +906,7 @@ :resource_boundary: :cpu :weight: 3 :idempotent: + :tags: [] - :name: pipeline_default:pipeline_metrics :feature_category: :continuous_integration :has_external_dependencies: @@ -766,6 +914,7 @@ :resource_boundary: :unknown :weight: 3 :idempotent: + :tags: [] - :name: pipeline_default:pipeline_notification :feature_category: :continuous_integration :has_external_dependencies: @@ -773,6 +922,7 @@ :resource_boundary: :cpu :weight: 3 :idempotent: + :tags: [] - :name: pipeline_default:pipeline_update_ci_ref_status :feature_category: :continuous_integration :has_external_dependencies: @@ -780,6 +930,7 @@ :resource_boundary: :cpu :weight: 3 :idempotent: + :tags: [] - :name: pipeline_hooks:build_hooks :feature_category: :continuous_integration :has_external_dependencies: @@ -787,6 +938,7 @@ :resource_boundary: :unknown :weight: 2 :idempotent: + :tags: [] - :name: pipeline_hooks:pipeline_hooks :feature_category: :continuous_integration :has_external_dependencies: @@ -794,6 +946,7 @@ :resource_boundary: :cpu :weight: 2 :idempotent: + :tags: [] - :name: pipeline_processing:build_finished :feature_category: :continuous_integration :has_external_dependencies: @@ -801,6 +954,7 @@ :resource_boundary: :cpu :weight: 5 :idempotent: + :tags: [] - :name: pipeline_processing:build_queue :feature_category: :continuous_integration :has_external_dependencies: @@ -808,6 +962,7 @@ :resource_boundary: :cpu :weight: 5 :idempotent: + :tags: [] - :name: pipeline_processing:build_success :feature_category: :continuous_integration :has_external_dependencies: @@ -815,6 +970,7 @@ :resource_boundary: :unknown :weight: 5 :idempotent: + :tags: [] - :name: pipeline_processing:ci_build_prepare :feature_category: :continuous_integration :has_external_dependencies: @@ -822,6 +978,7 @@ :resource_boundary: :unknown :weight: 5 :idempotent: + :tags: [] - :name: pipeline_processing:ci_build_schedule :feature_category: :continuous_integration :has_external_dependencies: @@ -829,6 +986,7 @@ :resource_boundary: :cpu :weight: 5 :idempotent: + :tags: [] - :name: pipeline_processing:ci_resource_groups_assign_resource_from_resource_group :feature_category: :continuous_delivery :has_external_dependencies: @@ -836,6 +994,7 @@ :resource_boundary: :unknown :weight: 5 :idempotent: + :tags: [] - :name: pipeline_processing:pipeline_process :feature_category: :continuous_integration :has_external_dependencies: @@ -843,6 +1002,7 @@ :resource_boundary: :unknown :weight: 5 :idempotent: + :tags: [] - :name: pipeline_processing:pipeline_update :feature_category: :continuous_integration :has_external_dependencies: @@ -850,6 +1010,7 @@ :resource_boundary: :unknown :weight: 5 :idempotent: + :tags: [] - :name: pipeline_processing:stage_update :feature_category: :continuous_integration :has_external_dependencies: @@ -857,6 +1018,7 @@ :resource_boundary: :unknown :weight: 5 :idempotent: true + :tags: [] - :name: pipeline_processing:update_head_pipeline_for_merge_request :feature_category: :continuous_integration :has_external_dependencies: @@ -864,6 +1026,7 @@ :resource_boundary: :cpu :weight: 5 :idempotent: true + :tags: [] - :name: repository_check:repository_check_batch :feature_category: :source_code_management :has_external_dependencies: @@ -871,6 +1034,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: + :tags: [] - :name: repository_check:repository_check_clear :feature_category: :source_code_management :has_external_dependencies: @@ -878,6 +1042,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: + :tags: [] - :name: repository_check:repository_check_single_repository :feature_category: :source_code_management :has_external_dependencies: @@ -885,6 +1050,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: + :tags: [] - :name: todos_destroyer:todos_destroyer_confidential_issue :feature_category: :issue_tracking :has_external_dependencies: @@ -892,6 +1058,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: + :tags: [] - :name: todos_destroyer:todos_destroyer_entity_leave :feature_category: :issue_tracking :has_external_dependencies: @@ -899,6 +1066,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: + :tags: [] - :name: todos_destroyer:todos_destroyer_group_private :feature_category: :issue_tracking :has_external_dependencies: @@ -906,6 +1074,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: + :tags: [] - :name: todos_destroyer:todos_destroyer_private_features :feature_category: :issue_tracking :has_external_dependencies: @@ -913,6 +1082,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: + :tags: [] - :name: todos_destroyer:todos_destroyer_project_private :feature_category: :issue_tracking :has_external_dependencies: @@ -920,6 +1090,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: + :tags: [] - :name: update_namespace_statistics:namespaces_root_statistics :feature_category: :source_code_management :has_external_dependencies: @@ -927,6 +1098,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: true + :tags: [] - :name: update_namespace_statistics:namespaces_schedule_aggregation :feature_category: :source_code_management :has_external_dependencies: @@ -934,6 +1106,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: true + :tags: [] - :name: authorized_keys :feature_category: :source_code_management :has_external_dependencies: @@ -941,6 +1114,7 @@ :resource_boundary: :unknown :weight: 2 :idempotent: true + :tags: [] - :name: authorized_projects :feature_category: :authentication_and_authorization :has_external_dependencies: @@ -948,6 +1122,7 @@ :resource_boundary: :unknown :weight: 2 :idempotent: true + :tags: [] - :name: background_migration :feature_category: :database :has_external_dependencies: @@ -955,6 +1130,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: + :tags: [] - :name: chat_notification :feature_category: :chatops :has_external_dependencies: true @@ -962,6 +1138,7 @@ :resource_boundary: :unknown :weight: 2 :idempotent: + :tags: [] - :name: create_commit_signature :feature_category: :source_code_management :has_external_dependencies: @@ -969,6 +1146,7 @@ :resource_boundary: :unknown :weight: 2 :idempotent: true + :tags: [] - :name: create_evidence :feature_category: :release_evidence :has_external_dependencies: @@ -976,6 +1154,7 @@ :resource_boundary: :unknown :weight: 2 :idempotent: + :tags: [] - :name: create_note_diff_file :feature_category: :source_code_management :has_external_dependencies: @@ -983,6 +1162,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: + :tags: [] - :name: default :feature_category: :has_external_dependencies: @@ -990,6 +1170,7 @@ :resource_boundary: :weight: 1 :idempotent: + :tags: [] - :name: delete_diff_files :feature_category: :source_code_management :has_external_dependencies: @@ -997,6 +1178,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: + :tags: [] - :name: delete_merged_branches :feature_category: :source_code_management :has_external_dependencies: @@ -1004,6 +1186,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: + :tags: [] - :name: delete_stored_files :feature_category: :not_owned :has_external_dependencies: @@ -1011,6 +1194,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: + :tags: [] - :name: delete_user :feature_category: :authentication_and_authorization :has_external_dependencies: @@ -1018,6 +1202,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: + :tags: [] - :name: design_management_new_version :feature_category: :design_management :has_external_dependencies: @@ -1025,6 +1210,7 @@ :resource_boundary: :memory :weight: 1 :idempotent: + :tags: [] - :name: detect_repository_languages :feature_category: :source_code_management :has_external_dependencies: @@ -1032,6 +1218,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: + :tags: [] - :name: email_receiver :feature_category: :issue_tracking :has_external_dependencies: @@ -1039,6 +1226,7 @@ :resource_boundary: :unknown :weight: 2 :idempotent: + :tags: [] - :name: emails_on_push :feature_category: :source_code_management :has_external_dependencies: @@ -1046,6 +1234,7 @@ :resource_boundary: :cpu :weight: 2 :idempotent: + :tags: [] - :name: error_tracking_issue_link :feature_category: :error_tracking :has_external_dependencies: true @@ -1053,6 +1242,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: + :tags: [] - :name: expire_build_instance_artifacts :feature_category: :continuous_integration :has_external_dependencies: @@ -1060,6 +1250,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: + :tags: [] - :name: export_csv :feature_category: :issue_tracking :has_external_dependencies: @@ -1067,6 +1258,7 @@ :resource_boundary: :cpu :weight: 1 :idempotent: + :tags: [] - :name: external_service_reactive_caching :feature_category: :not_owned :has_external_dependencies: true @@ -1074,6 +1266,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: + :tags: [] - :name: file_hook :feature_category: :integrations :has_external_dependencies: @@ -1081,6 +1274,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: + :tags: [] - :name: git_garbage_collect :feature_category: :gitaly :has_external_dependencies: @@ -1088,6 +1282,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: + :tags: [] - :name: github_import_advance_stage :feature_category: :importers :has_external_dependencies: @@ -1095,6 +1290,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: + :tags: [] - :name: gitlab_shell :feature_category: :source_code_management :has_external_dependencies: @@ -1102,6 +1298,7 @@ :resource_boundary: :unknown :weight: 2 :idempotent: + :tags: [] - :name: group_destroy :feature_category: :subgroups :has_external_dependencies: @@ -1109,6 +1306,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: + :tags: [] - :name: group_export :feature_category: :importers :has_external_dependencies: @@ -1116,6 +1314,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: + :tags: [] - :name: group_import :feature_category: :importers :has_external_dependencies: @@ -1123,6 +1322,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: + :tags: [] - :name: import_issues_csv :feature_category: :issue_tracking :has_external_dependencies: @@ -1130,6 +1330,7 @@ :resource_boundary: :cpu :weight: 2 :idempotent: + :tags: [] - :name: invalid_gpg_signature_update :feature_category: :source_code_management :has_external_dependencies: @@ -1137,6 +1338,7 @@ :resource_boundary: :unknown :weight: 2 :idempotent: + :tags: [] - :name: irker :feature_category: :integrations :has_external_dependencies: @@ -1144,6 +1346,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: + :tags: [] - :name: mailers :feature_category: :has_external_dependencies: @@ -1151,6 +1354,7 @@ :resource_boundary: :weight: 2 :idempotent: + :tags: [] - :name: merge :feature_category: :source_code_management :has_external_dependencies: @@ -1158,6 +1362,7 @@ :resource_boundary: :unknown :weight: 5 :idempotent: + :tags: [] - :name: merge_request_mergeability_check :feature_category: :source_code_management :has_external_dependencies: @@ -1165,6 +1370,15 @@ :resource_boundary: :unknown :weight: 1 :idempotent: true + :tags: [] +- :name: metrics_dashboard_prune_old_annotations + :feature_category: :metrics + :has_external_dependencies: + :urgency: :low + :resource_boundary: :unknown + :weight: 1 + :idempotent: true + :tags: [] - :name: migrate_external_diffs :feature_category: :source_code_management :has_external_dependencies: @@ -1172,6 +1386,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: + :tags: [] - :name: namespaceless_project_destroy :feature_category: :authentication_and_authorization :has_external_dependencies: @@ -1179,6 +1394,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: + :tags: [] - :name: new_issue :feature_category: :issue_tracking :has_external_dependencies: @@ -1186,6 +1402,7 @@ :resource_boundary: :cpu :weight: 2 :idempotent: + :tags: [] - :name: new_merge_request :feature_category: :source_code_management :has_external_dependencies: @@ -1193,6 +1410,7 @@ :resource_boundary: :cpu :weight: 2 :idempotent: + :tags: [] - :name: new_note :feature_category: :issue_tracking :has_external_dependencies: @@ -1200,6 +1418,7 @@ :resource_boundary: :cpu :weight: 2 :idempotent: + :tags: [] - :name: pages :feature_category: :pages :has_external_dependencies: @@ -1207,6 +1426,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: + :tags: [] - :name: pages_domain_ssl_renewal :feature_category: :pages :has_external_dependencies: @@ -1214,6 +1434,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: + :tags: [] - :name: pages_domain_verification :feature_category: :pages :has_external_dependencies: @@ -1221,6 +1442,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: + :tags: [] - :name: phabricator_import_import_tasks :feature_category: :importers :has_external_dependencies: @@ -1228,6 +1450,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: + :tags: [] - :name: post_receive :feature_category: :source_code_management :has_external_dependencies: @@ -1235,6 +1458,7 @@ :resource_boundary: :cpu :weight: 5 :idempotent: + :tags: [] - :name: process_commit :feature_category: :source_code_management :has_external_dependencies: @@ -1242,6 +1466,7 @@ :resource_boundary: :unknown :weight: 3 :idempotent: true + :tags: [] - :name: project_cache :feature_category: :source_code_management :has_external_dependencies: @@ -1249,6 +1474,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: + :tags: [] - :name: project_daily_statistics :feature_category: :source_code_management :has_external_dependencies: @@ -1256,6 +1482,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: + :tags: [] - :name: project_destroy :feature_category: :source_code_management :has_external_dependencies: @@ -1263,6 +1490,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: + :tags: [] - :name: project_export :feature_category: :importers :has_external_dependencies: @@ -1270,6 +1498,7 @@ :resource_boundary: :memory :weight: 1 :idempotent: + :tags: [] - :name: project_service :feature_category: :integrations :has_external_dependencies: true @@ -1277,6 +1506,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: + :tags: [] - :name: project_update_repository_storage :feature_category: :gitaly :has_external_dependencies: @@ -1284,6 +1514,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: + :tags: [] - :name: prometheus_create_default_alerts :feature_category: :incident_management :has_external_dependencies: @@ -1291,6 +1522,15 @@ :resource_boundary: :unknown :weight: 1 :idempotent: true + :tags: [] +- :name: propagate_integration + :feature_category: :integrations + :has_external_dependencies: + :urgency: :low + :resource_boundary: :unknown + :weight: 1 + :idempotent: true + :tags: [] - :name: propagate_service_template :feature_category: :source_code_management :has_external_dependencies: @@ -1298,6 +1538,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: + :tags: [] - :name: reactive_caching :feature_category: :not_owned :has_external_dependencies: @@ -1305,6 +1546,7 @@ :resource_boundary: :cpu :weight: 1 :idempotent: + :tags: [] - :name: rebase :feature_category: :source_code_management :has_external_dependencies: @@ -1312,6 +1554,7 @@ :resource_boundary: :unknown :weight: 2 :idempotent: + :tags: [] - :name: remote_mirror_notification :feature_category: :source_code_management :has_external_dependencies: @@ -1319,6 +1562,7 @@ :resource_boundary: :unknown :weight: 2 :idempotent: + :tags: [] - :name: repository_cleanup :feature_category: :source_code_management :has_external_dependencies: @@ -1326,6 +1570,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: + :tags: [] - :name: repository_fork :feature_category: :source_code_management :has_external_dependencies: @@ -1333,6 +1578,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: + :tags: [] - :name: repository_import :feature_category: :importers :has_external_dependencies: true @@ -1340,6 +1586,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: + :tags: [] - :name: repository_remove_remote :feature_category: :source_code_management :has_external_dependencies: @@ -1347,6 +1594,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: + :tags: [] - :name: repository_update_remote_mirror :feature_category: :source_code_management :has_external_dependencies: true @@ -1354,6 +1602,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: + :tags: [] - :name: self_monitoring_project_create :feature_category: :metrics :has_external_dependencies: @@ -1361,6 +1610,7 @@ :resource_boundary: :unknown :weight: 2 :idempotent: + :tags: [] - :name: self_monitoring_project_delete :feature_category: :metrics :has_external_dependencies: @@ -1368,6 +1618,7 @@ :resource_boundary: :unknown :weight: 2 :idempotent: + :tags: [] - :name: system_hook_push :feature_category: :source_code_management :has_external_dependencies: @@ -1375,6 +1626,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: + :tags: [] - :name: update_external_pull_requests :feature_category: :source_code_management :has_external_dependencies: @@ -1382,6 +1634,7 @@ :resource_boundary: :unknown :weight: 3 :idempotent: + :tags: [] - :name: update_highest_role :feature_category: :authentication_and_authorization :has_external_dependencies: @@ -1389,6 +1642,7 @@ :resource_boundary: :unknown :weight: 2 :idempotent: true + :tags: [] - :name: update_merge_requests :feature_category: :source_code_management :has_external_dependencies: @@ -1396,6 +1650,7 @@ :resource_boundary: :cpu :weight: 3 :idempotent: + :tags: [] - :name: update_project_statistics :feature_category: :source_code_management :has_external_dependencies: @@ -1403,6 +1658,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: + :tags: [] - :name: upload_checksum :feature_category: :geo_replication :has_external_dependencies: @@ -1410,6 +1666,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: + :tags: [] - :name: web_hook :feature_category: :integrations :has_external_dependencies: true @@ -1417,6 +1674,7 @@ :resource_boundary: :unknown :weight: 1 :idempotent: + :tags: [] - :name: x509_certificate_revoke :feature_category: :source_code_management :has_external_dependencies: @@ -1424,3 +1682,4 @@ :resource_boundary: :unknown :weight: 1 :idempotent: true + :tags: [] diff --git a/app/workers/authorized_keys_worker.rb b/app/workers/authorized_keys_worker.rb index eabfd89ba60..ab0e7fc4921 100644 --- a/app/workers/authorized_keys_worker.rb +++ b/app/workers/authorized_keys_worker.rb @@ -9,6 +9,7 @@ class AuthorizedKeysWorker urgency :high weight 2 idempotent! + loggable_arguments 0 def perform(action, *args) return unless Gitlab::CurrentSettings.authorized_keys_enabled? diff --git a/app/workers/authorized_project_update/user_refresh_with_low_urgency_worker.rb b/app/workers/authorized_project_update/user_refresh_with_low_urgency_worker.rb index 19038cb8900..7ca59a72adf 100644 --- a/app/workers/authorized_project_update/user_refresh_with_low_urgency_worker.rb +++ b/app/workers/authorized_project_update/user_refresh_with_low_urgency_worker.rb @@ -5,6 +5,7 @@ module AuthorizedProjectUpdate feature_category :authentication_and_authorization urgency :low queue_namespace :authorized_project_update + deduplicate :until_executing, including_scheduled: true idempotent! end diff --git a/app/workers/authorized_projects_worker.rb b/app/workers/authorized_projects_worker.rb index a35e0320553..62eea8e0462 100644 --- a/app/workers/authorized_projects_worker.rb +++ b/app/workers/authorized_projects_worker.rb @@ -7,8 +7,8 @@ class AuthorizedProjectsWorker feature_category :authentication_and_authorization urgency :high weight 2 - idempotent! + loggable_arguments 1 # For the job waiter key # This is a workaround for a Ruby 2.3.7 bug. rspec-mocks cannot restore the # visibility of prepended modules. See https://github.com/rspec/rspec-mocks/issues/1231 diff --git a/app/workers/background_migration_worker.rb b/app/workers/background_migration_worker.rb index 6a64afe47de..bff864ba420 100644 --- a/app/workers/background_migration_worker.rb +++ b/app/workers/background_migration_worker.rb @@ -5,6 +5,7 @@ class BackgroundMigrationWorker # rubocop:disable Scalability/IdempotentWorker feature_category :database urgency :throttled + loggable_arguments 0, 1 # The minimum amount of time between processing two jobs of the same migration # class. diff --git a/app/workers/build_finished_worker.rb b/app/workers/build_finished_worker.rb index b6ef9ab4710..d38780dd08d 100644 --- a/app/workers/build_finished_worker.rb +++ b/app/workers/build_finished_worker.rb @@ -28,6 +28,7 @@ class BuildFinishedWorker # rubocop:disable Scalability/IdempotentWorker # We execute these in sync to reduce IO. BuildTraceSectionsWorker.new.perform(build.id) BuildCoverageWorker.new.perform(build.id) + Ci::BuildReportResultWorker.new.perform(build.id) # We execute these async as these are independent operations. BuildHooksWorker.perform_async(build.id) diff --git a/app/workers/ci/build_report_result_worker.rb b/app/workers/ci/build_report_result_worker.rb new file mode 100644 index 00000000000..60387936d0b --- /dev/null +++ b/app/workers/ci/build_report_result_worker.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +module Ci + class BuildReportResultWorker + include ApplicationWorker + include PipelineBackgroundQueue + + idempotent! + + def perform(build_id) + Ci::Build.find_by_id(build_id).try do |build| + Ci::BuildReportResultService.new.execute(build) + end + end + end +end diff --git a/app/workers/cleanup_container_repository_worker.rb b/app/workers/cleanup_container_repository_worker.rb index c3fac453e73..4469ea8cff9 100644 --- a/app/workers/cleanup_container_repository_worker.rb +++ b/app/workers/cleanup_container_repository_worker.rb @@ -5,6 +5,7 @@ class CleanupContainerRepositoryWorker # rubocop:disable Scalability/IdempotentW queue_namespace :container_repository feature_category :container_registry + loggable_arguments 2 attr_reader :container_repository, :current_user diff --git a/app/workers/cluster_install_app_worker.rb b/app/workers/cluster_install_app_worker.rb index 002932a0fa5..f3da4d5c4bb 100644 --- a/app/workers/cluster_install_app_worker.rb +++ b/app/workers/cluster_install_app_worker.rb @@ -6,6 +6,7 @@ class ClusterInstallAppWorker # rubocop:disable Scalability/IdempotentWorker include ClusterApplications worker_has_external_dependencies! + loggable_arguments 0 def perform(app_name, app_id) find_application(app_name, app_id) do |app| diff --git a/app/workers/cluster_patch_app_worker.rb b/app/workers/cluster_patch_app_worker.rb index f75004aa3e5..b0393809802 100644 --- a/app/workers/cluster_patch_app_worker.rb +++ b/app/workers/cluster_patch_app_worker.rb @@ -6,6 +6,7 @@ class ClusterPatchAppWorker # rubocop:disable Scalability/IdempotentWorker include ClusterApplications worker_has_external_dependencies! + loggable_arguments 0 def perform(app_name, app_id) find_application(app_name, app_id) do |app| diff --git a/app/workers/cluster_update_app_worker.rb b/app/workers/cluster_update_app_worker.rb index 7ceeb167b33..29feb813043 100644 --- a/app/workers/cluster_update_app_worker.rb +++ b/app/workers/cluster_update_app_worker.rb @@ -9,6 +9,7 @@ class ClusterUpdateAppWorker # rubocop:disable Scalability/IdempotentWorker include ExclusiveLeaseGuard sidekiq_options retry: 3, dead: false + loggable_arguments 0, 3 LEASE_TIMEOUT = 10.minutes.to_i diff --git a/app/workers/cluster_upgrade_app_worker.rb b/app/workers/cluster_upgrade_app_worker.rb index 99f48415f08..d4650ab3a85 100644 --- a/app/workers/cluster_upgrade_app_worker.rb +++ b/app/workers/cluster_upgrade_app_worker.rb @@ -6,6 +6,7 @@ class ClusterUpgradeAppWorker # rubocop:disable Scalability/IdempotentWorker include ClusterApplications worker_has_external_dependencies! + loggable_arguments 0 def perform(app_name, app_id) find_application(app_name, app_id) do |app| diff --git a/app/workers/cluster_wait_for_app_installation_worker.rb b/app/workers/cluster_wait_for_app_installation_worker.rb index e098c3b86b5..4bc29807ea4 100644 --- a/app/workers/cluster_wait_for_app_installation_worker.rb +++ b/app/workers/cluster_wait_for_app_installation_worker.rb @@ -10,6 +10,7 @@ class ClusterWaitForAppInstallationWorker # rubocop:disable Scalability/Idempote worker_has_external_dependencies! worker_resource_boundary :cpu + loggable_arguments 0 def perform(app_name, app_id) find_application(app_name, app_id) do |app| diff --git a/app/workers/cluster_wait_for_app_update_worker.rb b/app/workers/cluster_wait_for_app_update_worker.rb index 9f1d83c2c7b..c0a11eb93a7 100644 --- a/app/workers/cluster_wait_for_app_update_worker.rb +++ b/app/workers/cluster_wait_for_app_update_worker.rb @@ -8,6 +8,8 @@ class ClusterWaitForAppUpdateWorker # rubocop:disable Scalability/IdempotentWork INTERVAL = 10.seconds TIMEOUT = 20.minutes + loggable_arguments 0 + def perform(app_name, app_id) find_application(app_name, app_id) do |app| ::Clusters::Applications::CheckUpgradeProgressService.new(app).execute diff --git a/app/workers/cluster_wait_for_ingress_ip_address_worker.rb b/app/workers/cluster_wait_for_ingress_ip_address_worker.rb index c7336ee515d..fa46135d279 100644 --- a/app/workers/cluster_wait_for_ingress_ip_address_worker.rb +++ b/app/workers/cluster_wait_for_ingress_ip_address_worker.rb @@ -6,6 +6,7 @@ class ClusterWaitForIngressIpAddressWorker # rubocop:disable Scalability/Idempot include ClusterApplications worker_has_external_dependencies! + loggable_arguments 0 def perform(app_name, app_id) find_application(app_name, app_id) do |app| diff --git a/app/workers/clusters/applications/activate_service_worker.rb b/app/workers/clusters/applications/activate_service_worker.rb index abd7f8eaea4..c92f978a7d2 100644 --- a/app/workers/clusters/applications/activate_service_worker.rb +++ b/app/workers/clusters/applications/activate_service_worker.rb @@ -6,6 +6,8 @@ module Clusters include ApplicationWorker include ClusterQueue + loggable_arguments 1 + def perform(cluster_id, service_name) cluster = Clusters::Cluster.find_by_id(cluster_id) return unless cluster diff --git a/app/workers/clusters/applications/check_prometheus_health_worker.rb b/app/workers/clusters/applications/check_prometheus_health_worker.rb new file mode 100644 index 00000000000..2e8ee739946 --- /dev/null +++ b/app/workers/clusters/applications/check_prometheus_health_worker.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +module Clusters + module Applications + class CheckPrometheusHealthWorker + include ApplicationWorker + # rubocop:disable Scalability/CronWorkerContext + # This worker does not perform work scoped to a context + include CronjobQueue + # rubocop:enable Scalability/CronWorkerContext + + queue_namespace :incident_management + feature_category :incident_management + urgency :low + + idempotent! + worker_has_external_dependencies! + + def perform + demo_project_ids = Gitlab::Monitor::DemoProjects.primary_keys + + clusters = Clusters::Cluster.with_application_prometheus + .with_project_alert_service_data(demo_project_ids) + + # Move to a seperate worker with scoped context if expanded to do work on customer projects + clusters.each { |cluster| Clusters::Applications::PrometheusHealthCheckService.new(cluster).execute } + end + end + end +end diff --git a/app/workers/clusters/applications/deactivate_service_worker.rb b/app/workers/clusters/applications/deactivate_service_worker.rb index fecbb6dde45..4d103bb0edc 100644 --- a/app/workers/clusters/applications/deactivate_service_worker.rb +++ b/app/workers/clusters/applications/deactivate_service_worker.rb @@ -6,6 +6,8 @@ module Clusters include ApplicationWorker include ClusterQueue + loggable_arguments 1 + def perform(cluster_id, service_name) cluster = Clusters::Cluster.find_by_id(cluster_id) raise cluster_missing_error(service_name) unless cluster diff --git a/app/workers/clusters/applications/uninstall_worker.rb b/app/workers/clusters/applications/uninstall_worker.rb index 977a25e8442..a9307931b59 100644 --- a/app/workers/clusters/applications/uninstall_worker.rb +++ b/app/workers/clusters/applications/uninstall_worker.rb @@ -8,6 +8,7 @@ module Clusters include ClusterApplications worker_has_external_dependencies! + loggable_arguments 0 def perform(app_name, app_id) find_application(app_name, app_id) do |app| diff --git a/app/workers/clusters/applications/wait_for_uninstall_app_worker.rb b/app/workers/clusters/applications/wait_for_uninstall_app_worker.rb index a486cfa90b7..dc842788374 100644 --- a/app/workers/clusters/applications/wait_for_uninstall_app_worker.rb +++ b/app/workers/clusters/applications/wait_for_uninstall_app_worker.rb @@ -12,6 +12,7 @@ module Clusters worker_has_external_dependencies! worker_resource_boundary :cpu + loggable_arguments 0 def perform(app_name, app_id) find_application(app_name, app_id) do |app| diff --git a/app/workers/concerns/application_worker.rb b/app/workers/concerns/application_worker.rb index 7ab9a0c2a02..9c942228111 100644 --- a/app/workers/concerns/application_worker.rb +++ b/app/workers/concerns/application_worker.rb @@ -76,6 +76,22 @@ module ApplicationWorker get_sidekiq_options['queue'].to_s end + # Set/get which arguments can be logged and sent to Sentry. + # + # Numeric arguments are logged by default, so there is no need to + # list those. + # + # Non-numeric arguments must be listed by position, as Sidekiq + # cannot see argument names. + # + def loggable_arguments(*args) + if args.any? + @loggable_arguments = args + else + @loggable_arguments || [] + end + end + def queue_size Sidekiq::Queue.new(queue).size end @@ -84,7 +100,7 @@ module ApplicationWorker Sidekiq::Client.push_bulk('class' => self, 'args' => args_list) end - def bulk_perform_in(delay, args_list) + def bulk_perform_in(delay, args_list, batch_size: nil, batch_delay: nil) now = Time.now.to_i schedule = now + delay.to_i @@ -92,7 +108,14 @@ module ApplicationWorker raise ArgumentError, _('The schedule time must be in the future!') end - Sidekiq::Client.push_bulk('class' => self, 'args' => args_list, 'at' => schedule) + if batch_size && batch_delay + args_list.each_slice(batch_size.to_i).with_index do |args_batch, idx| + batch_schedule = schedule + idx * batch_delay.to_i + Sidekiq::Client.push_bulk('class' => self, 'args' => args_batch, 'at' => batch_schedule) + end + else + Sidekiq::Client.push_bulk('class' => self, 'args' => args_list, 'at' => schedule) + end end end end diff --git a/app/workers/concerns/gitlab/github_import/queue.rb b/app/workers/concerns/gitlab/github_import/queue.rb index 7cc23dd7c0b..05eb7fbc2cb 100644 --- a/app/workers/concerns/gitlab/github_import/queue.rb +++ b/app/workers/concerns/gitlab/github_import/queue.rb @@ -15,6 +15,17 @@ module Gitlab # this is better than a project being stuck in the "import" state # forever. sidekiq_options dead: false, retry: 5 + + sidekiq_retries_exhausted do |msg, e| + Gitlab::Import::Logger.error( + event: :github_importer_exhausted, + message: msg['error_message'], + class: msg['class'], + args: msg['args'], + exception_message: e.message, + exception_backtrace: e.backtrace + ) + end end end end diff --git a/app/workers/concerns/gitlab/jira_import/import_worker.rb b/app/workers/concerns/gitlab/jira_import/import_worker.rb index 537300e6eba..fdc6e64bbaa 100644 --- a/app/workers/concerns/gitlab/jira_import/import_worker.rb +++ b/app/workers/concerns/gitlab/jira_import/import_worker.rb @@ -7,6 +7,7 @@ module Gitlab included do include ApplicationWorker + include ProjectImportOptions include Gitlab::JiraImport::QueueOptions end @@ -26,7 +27,6 @@ module Gitlab def can_import?(project) return false unless project - return false unless project.jira_issues_import_feature_flag_enabled? project.latest_jira_import&.started? end diff --git a/app/workers/concerns/project_import_options.rb b/app/workers/concerns/project_import_options.rb index c8ee5539441..4df209da29d 100644 --- a/app/workers/concerns/project_import_options.rb +++ b/app/workers/concerns/project_import_options.rb @@ -6,7 +6,7 @@ module ProjectImportOptions IMPORT_RETRY_COUNT = 5 included do - sidekiq_options retry: IMPORT_RETRY_COUNT, status_expiration: StuckImportJobsWorker::IMPORT_JOBS_EXPIRATION + sidekiq_options retry: IMPORT_RETRY_COUNT, status_expiration: Gitlab::Import::StuckImportJob::IMPORT_JOBS_EXPIRATION # We only want to mark the project as failed once we exhausted all retries sidekiq_retries_exhausted do |job| diff --git a/app/workers/concerns/reactive_cacheable_worker.rb b/app/workers/concerns/reactive_cacheable_worker.rb index e73707c2b43..189b0607605 100644 --- a/app/workers/concerns/reactive_cacheable_worker.rb +++ b/app/workers/concerns/reactive_cacheable_worker.rb @@ -7,6 +7,7 @@ module ReactiveCacheableWorker include ApplicationWorker feature_category_not_owned! + loggable_arguments 0 def self.context_for_arguments(arguments) class_name, *_other_args = arguments diff --git a/app/workers/concerns/worker_attributes.rb b/app/workers/concerns/worker_attributes.rb index b60179531af..b19217b15de 100644 --- a/app/workers/concerns/worker_attributes.rb +++ b/app/workers/concerns/worker_attributes.rb @@ -111,6 +111,28 @@ module WorkerAttributes 1 end + def tags(*values) + worker_attributes[:tags] = values + end + + def get_tags + Array(worker_attributes[:tags]) + end + + def deduplicate(strategy, options = {}) + worker_attributes[:deduplication_strategy] = strategy + worker_attributes[:deduplication_options] = options + end + + def get_deduplicate_strategy + worker_attributes[:deduplication_strategy] || + Gitlab::SidekiqMiddleware::DuplicateJobs::DuplicateJob::DEFAULT_STRATEGY + end + + def get_deduplication_options + worker_attributes[:deduplication_options] || {} + end + protected # Returns a worker attribute declared on this class or its parent class. diff --git a/app/workers/container_expiration_policy_worker.rb b/app/workers/container_expiration_policy_worker.rb index e1544be5aed..96590e165ae 100644 --- a/app/workers/container_expiration_policy_worker.rb +++ b/app/workers/container_expiration_policy_worker.rb @@ -12,6 +12,8 @@ class ContainerExpirationPolicyWorker # rubocop:disable Scalability/IdempotentWo user: container_expiration_policy.project.owner) do |project:, user:| ContainerExpirationPolicyService.new(project, user) .execute(container_expiration_policy) + rescue ContainerExpirationPolicyService::InvalidPolicyError => e + Gitlab::ErrorTracking.log_exception(e, container_expiration_policy_id: container_expiration_policy.id) end end end diff --git a/app/workers/create_commit_signature_worker.rb b/app/workers/create_commit_signature_worker.rb index a88d2bf7d15..f81baf20d19 100644 --- a/app/workers/create_commit_signature_worker.rb +++ b/app/workers/create_commit_signature_worker.rb @@ -5,8 +5,8 @@ class CreateCommitSignatureWorker feature_category :source_code_management weight 2 - idempotent! + loggable_arguments 0 # rubocop: disable CodeReuse/ActiveRecord def perform(commit_shas, project_id) @@ -37,7 +37,7 @@ class CreateCommitSignatureWorker commits.each do |commit| commit&.signature rescue => e - Rails.logger.error("Failed to create signature for commit #{commit.id}. Error: #{e.message}") # rubocop:disable Gitlab/RailsLogger + Gitlab::AppLogger.error("Failed to create signature for commit #{commit.id}. Error: #{e.message}") end end # rubocop: enable CodeReuse/ActiveRecord diff --git a/app/workers/create_evidence_worker.rb b/app/workers/create_evidence_worker.rb index 135e2ac38b4..b18028e4114 100644 --- a/app/workers/create_evidence_worker.rb +++ b/app/workers/create_evidence_worker.rb @@ -6,10 +6,15 @@ class CreateEvidenceWorker # rubocop:disable Scalability/IdempotentWorker feature_category :release_evidence weight 2 - def perform(release_id) + # pipeline_id is optional for backward compatibility with existing jobs + # caller should always try to provide the pipeline and pass nil only + # if pipeline is absent + def perform(release_id, pipeline_id = nil) release = Release.find_by_id(release_id) return unless release - Releases::Evidence.create!(release: release) + pipeline = Ci::Pipeline.find_by_id(pipeline_id) + + ::Releases::CreateEvidenceService.new(release, pipeline: pipeline).execute end end diff --git a/app/workers/create_pipeline_worker.rb b/app/workers/create_pipeline_worker.rb index 54698518e4f..68fe44d01ce 100644 --- a/app/workers/create_pipeline_worker.rb +++ b/app/workers/create_pipeline_worker.rb @@ -8,6 +8,7 @@ class CreatePipelineWorker # rubocop:disable Scalability/IdempotentWorker feature_category :continuous_integration urgency :high worker_resource_boundary :cpu + loggable_arguments 2, 3, 4 def perform(project_id, user_id, ref, source, params = {}) project = Project.find(project_id) diff --git a/app/workers/delete_stored_files_worker.rb b/app/workers/delete_stored_files_worker.rb index 463f26fdd5a..9cf5631b7d8 100644 --- a/app/workers/delete_stored_files_worker.rb +++ b/app/workers/delete_stored_files_worker.rb @@ -4,6 +4,7 @@ class DeleteStoredFilesWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker feature_category_not_owned! + loggable_arguments 0 def perform(class_name, keys) klass = begin diff --git a/app/workers/delete_user_worker.rb b/app/workers/delete_user_worker.rb index d3b87c133d3..ed2e00f1241 100644 --- a/app/workers/delete_user_worker.rb +++ b/app/workers/delete_user_worker.rb @@ -4,6 +4,7 @@ class DeleteUserWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker feature_category :authentication_and_authorization + loggable_arguments 2 def perform(current_user_id, delete_user_id, options = {}) delete_user = User.find(delete_user_id) @@ -11,6 +12,6 @@ class DeleteUserWorker # rubocop:disable Scalability/IdempotentWorker Users::DestroyService.new(current_user).execute(delete_user, options.symbolize_keys) rescue Gitlab::Access::AccessDeniedError => e - Rails.logger.warn("User could not be destroyed: #{e}") # rubocop:disable Gitlab/RailsLogger + Gitlab::AppLogger.warn("User could not be destroyed: #{e}") end end diff --git a/app/workers/email_receiver_worker.rb b/app/workers/email_receiver_worker.rb index fcb88982c0b..9ceab9bb878 100644 --- a/app/workers/email_receiver_worker.rb +++ b/app/workers/email_receiver_worker.rb @@ -20,7 +20,7 @@ class EmailReceiverWorker # rubocop:disable Scalability/IdempotentWorker private def handle_failure(raw, error) - Rails.logger.warn("Email can not be processed: #{error}\n\n#{raw}") # rubocop:disable Gitlab/RailsLogger + Gitlab::AppLogger.warn("Email can not be processed: #{error}\n\n#{raw}") return unless raw.present? diff --git a/app/workers/expire_build_instance_artifacts_worker.rb b/app/workers/expire_build_instance_artifacts_worker.rb index 48fd086f88f..e6cd60a3e47 100644 --- a/app/workers/expire_build_instance_artifacts_worker.rb +++ b/app/workers/expire_build_instance_artifacts_worker.rb @@ -14,7 +14,7 @@ class ExpireBuildInstanceArtifactsWorker # rubocop:disable Scalability/Idempoten return unless build&.project && !build.project.pending_delete - Rails.logger.info "Removing artifacts for build #{build.id}..." # rubocop:disable Gitlab/RailsLogger + Gitlab::AppLogger.info("Removing artifacts for build #{build.id}...") build.erase_erasable_artifacts! end # rubocop: enable CodeReuse/ActiveRecord diff --git a/app/workers/export_csv_worker.rb b/app/workers/export_csv_worker.rb index 9e2b3ad9bb4..e7baaf40a41 100644 --- a/app/workers/export_csv_worker.rb +++ b/app/workers/export_csv_worker.rb @@ -5,6 +5,7 @@ class ExportCsvWorker # rubocop:disable Scalability/IdempotentWorker feature_category :issue_tracking worker_resource_boundary :cpu + loggable_arguments 2 def perform(current_user_id, project_id, params) @current_user = User.find(current_user_id) diff --git a/app/workers/file_hook_worker.rb b/app/workers/file_hook_worker.rb index f8cdea54a17..b1422cd8795 100644 --- a/app/workers/file_hook_worker.rb +++ b/app/workers/file_hook_worker.rb @@ -5,6 +5,7 @@ class FileHookWorker # rubocop:disable Scalability/IdempotentWorker sidekiq_options retry: false feature_category :integrations + loggable_arguments 0 def perform(file_name, data) success, message = Gitlab::FileHook.execute(file_name, data) diff --git a/app/workers/git_garbage_collect_worker.rb b/app/workers/git_garbage_collect_worker.rb index 37ca3af517f..f2222c7be5e 100644 --- a/app/workers/git_garbage_collect_worker.rb +++ b/app/workers/git_garbage_collect_worker.rb @@ -5,6 +5,7 @@ class GitGarbageCollectWorker # rubocop:disable Scalability/IdempotentWorker sidekiq_options retry: false feature_category :gitaly + loggable_arguments 1, 2, 3 # Timeout set to 24h LEASE_TIMEOUT = 86400 diff --git a/app/workers/gitlab/github_import/advance_stage_worker.rb b/app/workers/gitlab/github_import/advance_stage_worker.rb index 21e478f935b..834c2f7791c 100644 --- a/app/workers/gitlab/github_import/advance_stage_worker.rb +++ b/app/workers/gitlab/github_import/advance_stage_worker.rb @@ -12,6 +12,7 @@ module Gitlab sidekiq_options dead: false feature_category :importers + loggable_arguments 1, 2 # The known importer stages and their corresponding Sidekiq workers. STAGES = { diff --git a/app/workers/gitlab/import/stuck_import_job.rb b/app/workers/gitlab/import/stuck_import_job.rb new file mode 100644 index 00000000000..16be7a77ab1 --- /dev/null +++ b/app/workers/gitlab/import/stuck_import_job.rb @@ -0,0 +1,82 @@ +# frozen_string_literal: true + +module Gitlab + module Import + module StuckImportJob + extend ActiveSupport::Concern + + IMPORT_JOBS_EXPIRATION = 15.hours.seconds.to_i + + included do + include ApplicationWorker + # rubocop:disable Scalability/CronWorkerContext + # This worker updates several import states inline and does not schedule + # other jobs. So no context needed + include CronjobQueue + # rubocop:enable Scalability/CronWorkerContext + + feature_category :importers + worker_resource_boundary :cpu + end + + def perform + stuck_imports_without_jid_count = mark_imports_without_jid_as_failed! + stuck_imports_with_jid_count = mark_imports_with_jid_as_failed! + + track_metrics(stuck_imports_with_jid_count, stuck_imports_without_jid_count) + end + + private + + def track_metrics(with_jid_count, without_jid_count) + raise NotImplementedError + end + + def mark_imports_without_jid_as_failed! + enqueued_import_states_without_jid.each do |import_state| + import_state.mark_as_failed(error_message) + end.size + end + + def mark_imports_with_jid_as_failed! + jids_and_ids = enqueued_import_states_with_jid.pluck(:jid, :id).to_h # rubocop: disable CodeReuse/ActiveRecord + + # Find the jobs that aren't currently running or that exceeded the threshold. + completed_jids = Gitlab::SidekiqStatus.completed_jids(jids_and_ids.keys) + return 0 unless completed_jids.any? + + completed_import_state_ids = jids_and_ids.values_at(*completed_jids) + + # We select the import states again, because they may have transitioned from + # scheduled/started to finished/failed while we were looking up their Sidekiq status. + completed_import_states = enqueued_import_states_with_jid.id_in(completed_import_state_ids) + completed_import_state_jids = completed_import_states.map { |import_state| import_state.jid }.join(', ') + + Gitlab::Import::Logger.info( + message: 'Marked stuck import jobs as failed', + job_ids: completed_import_state_jids + ) + + completed_import_states.each do |import_state| + import_state.mark_as_failed(error_message) + end.size + end + + def enqueued_import_states + raise NotImplementedError + end + + def enqueued_import_states_with_jid + enqueued_import_states.with_jid + end + + def enqueued_import_states_without_jid + enqueued_import_states.without_jid + end + + def error_message + _("Import timed out. Import took longer than %{import_jobs_expiration} seconds") % { import_jobs_expiration: IMPORT_JOBS_EXPIRATION } + end + end + end +end diff --git a/app/workers/gitlab/import/stuck_project_import_jobs_worker.rb b/app/workers/gitlab/import/stuck_project_import_jobs_worker.rb new file mode 100644 index 00000000000..01979b2029f --- /dev/null +++ b/app/workers/gitlab/import/stuck_project_import_jobs_worker.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true +module Gitlab + module Import + class StuckProjectImportJobsWorker # rubocop:disable Scalability/IdempotentWorker + include Gitlab::Import::StuckImportJob + + private + + def track_metrics(with_jid_count, without_jid_count) + Gitlab::Metrics.add_event( + :stuck_import_jobs, + projects_without_jid_count: without_jid_count, + projects_with_jid_count: with_jid_count + ) + end + + def enqueued_import_states + ProjectImportState.with_status([:scheduled, :started]) + end + end + end +end diff --git a/app/workers/gitlab/jira_import/import_issue_worker.rb b/app/workers/gitlab/jira_import/import_issue_worker.rb index 78de5cf1307..7709d2ec31b 100644 --- a/app/workers/gitlab/jira_import/import_issue_worker.rb +++ b/app/workers/gitlab/jira_import/import_issue_worker.rb @@ -8,6 +8,8 @@ module Gitlab include Gitlab::JiraImport::QueueOptions include Gitlab::Import::DatabaseHelpers + loggable_arguments 3 + def perform(project_id, jira_issue_id, issue_attributes, waiter_key) issue_id = create_issue(issue_attributes, project_id) JiraImport.cache_issue_mapping(issue_id, jira_issue_id, project_id) @@ -48,7 +50,7 @@ module Gitlab label_link_attrs << build_label_attrs(issue_id, import_label_id.to_i) - Gitlab::Database.bulk_insert(LabelLink.table_name, label_link_attrs) + Gitlab::Database.bulk_insert(LabelLink.table_name, label_link_attrs) # rubocop:disable Gitlab/BulkInsert end def assign_issue(project_id, issue_id, assignee_ids) @@ -56,7 +58,7 @@ module Gitlab assignee_attrs = assignee_ids.map { |user_id| { issue_id: issue_id, user_id: user_id } } - Gitlab::Database.bulk_insert(IssueAssignee.table_name, assignee_attrs) + Gitlab::Database.bulk_insert(IssueAssignee.table_name, assignee_attrs) # rubocop:disable Gitlab/BulkInsert end def build_label_attrs(issue_id, label_id) diff --git a/app/workers/gitlab/jira_import/stage/start_import_worker.rb b/app/workers/gitlab/jira_import/stage/start_import_worker.rb index 5b36feadbd1..bfc02224ee4 100644 --- a/app/workers/gitlab/jira_import/stage/start_import_worker.rb +++ b/app/workers/gitlab/jira_import/stage/start_import_worker.rb @@ -25,7 +25,6 @@ module Gitlab def start_import return false unless project - return false unless project.jira_issues_import_feature_flag_enabled? return true if start(project.latest_jira_import) Gitlab::Import::Logger.info( diff --git a/app/workers/gitlab/jira_import/stuck_jira_import_jobs_worker.rb b/app/workers/gitlab/jira_import/stuck_jira_import_jobs_worker.rb new file mode 100644 index 00000000000..5e675193a8c --- /dev/null +++ b/app/workers/gitlab/jira_import/stuck_jira_import_jobs_worker.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module Gitlab + module JiraImport + class StuckJiraImportJobsWorker # rubocop:disable Scalability/IdempotentWorker + include Gitlab::Import::StuckImportJob + + private + + def track_metrics(with_jid_count, without_jid_count) + Gitlab::Metrics.add_event(:stuck_jira_import_jobs, + jira_imports_without_jid_count: with_jid_count, + jira_imports_with_jid_count: without_jid_count) + end + + def enqueued_import_states + JiraImportState.with_status([:scheduled, :started]) + end + end + end +end diff --git a/app/workers/gitlab/phabricator_import/base_worker.rb b/app/workers/gitlab/phabricator_import/base_worker.rb index 82ef9e825f9..2dc4855f854 100644 --- a/app/workers/gitlab/phabricator_import/base_worker.rb +++ b/app/workers/gitlab/phabricator_import/base_worker.rb @@ -13,7 +13,7 @@ # - It keeps track of the jobs so we know how many jobs are running for the # project # - It refreshes the import jid, so it doesn't get cleaned up by the -# `StuckImportJobsWorker` +# `Gitlab::Import::StuckProjectImportJobsWorker` # - It marks the import as failed if a job failed to many times # - It marks the import as finished when all remaining jobs are done module Gitlab diff --git a/app/workers/gitlab_shell_worker.rb b/app/workers/gitlab_shell_worker.rb index b104d3c681e..b8e1e3d8fc4 100644 --- a/app/workers/gitlab_shell_worker.rb +++ b/app/workers/gitlab_shell_worker.rb @@ -7,6 +7,7 @@ class GitlabShellWorker # rubocop:disable Scalability/IdempotentWorker feature_category :source_code_management urgency :high weight 2 + loggable_arguments 0 def perform(action, *arg) # Gitlab::Shell is being removed but we need to continue to process jobs diff --git a/app/workers/group_export_worker.rb b/app/workers/group_export_worker.rb index 3e0390429d6..6fd977e43d8 100644 --- a/app/workers/group_export_worker.rb +++ b/app/workers/group_export_worker.rb @@ -5,6 +5,7 @@ class GroupExportWorker # rubocop:disable Scalability/IdempotentWorker include ExceptionBacktrace feature_category :importers + loggable_arguments 2 def perform(current_user_id, group_id, params = {}) current_user = User.find(current_user_id) diff --git a/app/workers/group_import_worker.rb b/app/workers/group_import_worker.rb index d8f236013bf..36d81468d55 100644 --- a/app/workers/group_import_worker.rb +++ b/app/workers/group_import_worker.rb @@ -9,15 +9,16 @@ class GroupImportWorker # rubocop:disable Scalability/IdempotentWorker def perform(user_id, group_id) current_user = User.find(user_id) group = Group.find(group_id) - group_import = group.build_import_state(jid: self.jid) + group_import_state = group.import_state || group.build_import_state - group_import.start! + group_import_state.jid = self.jid + group_import_state.start! ::Groups::ImportExport::ImportService.new(group: group, user: current_user).execute - group_import.finish! + group_import_state.finish! rescue StandardError => e - group_import&.fail_op(e.message) + group_import_state&.fail_op(e.message) raise e end diff --git a/app/workers/hashed_storage/project_migrate_worker.rb b/app/workers/hashed_storage/project_migrate_worker.rb index 3ce60ce7eb6..03e53058dbb 100644 --- a/app/workers/hashed_storage/project_migrate_worker.rb +++ b/app/workers/hashed_storage/project_migrate_worker.rb @@ -5,6 +5,7 @@ module HashedStorage include ApplicationWorker queue_namespace :hashed_storage + loggable_arguments 1 attr_reader :project_id diff --git a/app/workers/hashed_storage/project_rollback_worker.rb b/app/workers/hashed_storage/project_rollback_worker.rb index 17b3cca83e1..d4a5e474323 100644 --- a/app/workers/hashed_storage/project_rollback_worker.rb +++ b/app/workers/hashed_storage/project_rollback_worker.rb @@ -5,6 +5,7 @@ module HashedStorage include ApplicationWorker queue_namespace :hashed_storage + loggable_arguments 1 attr_reader :project_id diff --git a/app/workers/incident_management/process_prometheus_alert_worker.rb b/app/workers/incident_management/process_prometheus_alert_worker.rb index 768e049c60e..e405bc2c2d2 100644 --- a/app/workers/incident_management/process_prometheus_alert_worker.rb +++ b/app/workers/incident_management/process_prometheus_alert_worker.rb @@ -41,23 +41,11 @@ module IncidentManagement end def find_gitlab_managed_event(alert) - payload_key = payload_key_for_alert(alert) - - PrometheusAlertEvent.find_by_payload_key(payload_key) + PrometheusAlertEvent.find_by_payload_key(alert.gitlab_fingerprint) end def find_self_managed_event(alert) - payload_key = payload_key_for_alert(alert) - - SelfManagedPrometheusAlertEvent.find_by_payload_key(payload_key) - end - - def payload_key_for_alert(alert) - if alert.gitlab_managed? - PrometheusAlertEvent.payload_key_for(alert.metric_id, alert.starts_at_raw) - else - SelfManagedPrometheusAlertEvent.payload_key_for(alert.starts_at_raw, alert.title, alert.full_query) - end + SelfManagedPrometheusAlertEvent.find_by_payload_key(alert.gitlab_fingerprint) end def create_issue(project, alert) diff --git a/app/workers/irker_worker.rb b/app/workers/irker_worker.rb index 09f53681e7f..687fb1cd02a 100644 --- a/app/workers/irker_worker.rb +++ b/app/workers/irker_worker.rb @@ -8,26 +8,30 @@ class IrkerWorker # rubocop:disable Scalability/IdempotentWorker feature_category :integrations - def perform(project_id, chans, colors, push_data, settings) - project = Project.find(project_id) + def perform(project_id, channels, colors, push_data, settings) + # Establish connection to irker server + return false unless start_connection(settings['server_host'], + settings['server_port']) - # Get config parameters - return false unless init_perform settings, chans, colors + @project = Project.find(project_id) + @colors = colors + @channels = channels - repo_name = push_data['repository']['name'] - committer = push_data['user_name'] - branch = push_data['ref'].gsub(%r'refs/[^/]*/', '') + @repo_path = @project.full_path + @repo_name = push_data['repository']['name'] + @committer = push_data['user_name'] + @branch = push_data['ref'].gsub(%r'refs/[^/]*/', '') if @colors - repo_name = "\x0304#{repo_name}\x0f" - branch = "\x0305#{branch}\x0f" + @repo_name = "\x0304#{@repo_name}\x0f" + @branch = "\x0305#{@branch}\x0f" end # First messages are for branch creation/deletion - send_branch_updates push_data, project, repo_name, committer, branch + send_branch_updates(push_data) # Next messages are for commits - send_commits push_data, project, repo_name, committer, branch + send_commits(push_data) close_connection true @@ -35,12 +39,6 @@ class IrkerWorker # rubocop:disable Scalability/IdempotentWorker private - def init_perform(set, chans, colors) - @colors = colors - @channels = chans - start_connection set['server_host'], set['server_port'] - end - def start_connection(irker_server, irker_port) begin @socket = TCPSocket.new irker_server, irker_port @@ -48,11 +46,13 @@ class IrkerWorker # rubocop:disable Scalability/IdempotentWorker logger.fatal "Can't connect to Irker daemon: #{e}" return false end + true end - def sendtoirker(privmsg) + def send_to_irker(privmsg) to_send = { to: @channels, privmsg: privmsg } + @socket.puts Gitlab::Json.dump(to_send) end @@ -60,51 +60,51 @@ class IrkerWorker # rubocop:disable Scalability/IdempotentWorker @socket.close end - def send_branch_updates(push_data, project, repo_name, committer, branch) - if Gitlab::Git.blank_ref?(push_data['before']) - send_new_branch project, repo_name, committer, branch - elsif Gitlab::Git.blank_ref?(push_data['after']) - send_del_branch repo_name, committer, branch - end + def send_branch_updates(push_data) + message = + if Gitlab::Git.blank_ref?(push_data['before']) + new_branch_message + elsif Gitlab::Git.blank_ref?(push_data['after']) + delete_branch_message + end + + send_to_irker(message) end - def send_new_branch(project, repo_name, committer, branch) - repo_path = project.full_path - newbranch = "#{Gitlab.config.gitlab.url}/#{repo_path}/-/branches" + def new_branch_message + newbranch = "#{Gitlab.config.gitlab.url}/#{@repo_path}/-/branches" newbranch = "\x0302\x1f#{newbranch}\x0f" if @colors - privmsg = "[#{repo_name}] #{committer} has created a new branch " \ - "#{branch}: #{newbranch}" - sendtoirker privmsg + "[#{@repo_name}] #{@committer} has created a new branch #{@branch}: #{newbranch}" end - def send_del_branch(repo_name, committer, branch) - privmsg = "[#{repo_name}] #{committer} has deleted the branch #{branch}" - sendtoirker privmsg + def delete_branch_message + "[#{@repo_name}] #{@committer} has deleted the branch #{@branch}" end - def send_commits(push_data, project, repo_name, committer, branch) + def send_commits(push_data) return if push_data['total_commits_count'] == 0 # Next message is for number of commit pushed, if any if Gitlab::Git.blank_ref?(push_data['before']) # Tweak on push_data["before"] in order to have a nice compare URL - push_data['before'] = before_on_new_branch push_data, project + push_data['before'] = before_on_new_branch(push_data) end - send_commits_count(push_data, project, repo_name, committer, branch) + send_commits_count(push_data) # One message per commit, limited by 3 messages (same limit as the # github irc hook) commits = push_data['commits'].first(3) - commits.each do |hook_attrs| - send_one_commit project, hook_attrs, repo_name, branch + commits.each do |commit_attrs| + send_one_commit(commit_attrs) end end - def before_on_new_branch(push_data, project) - commit = commit_from_id project, push_data['commits'][0]['id'] + def before_on_new_branch(push_data) + commit = commit_from_id(push_data['commits'][0]['id']) parents = commit.parents + # Return old value if there's no new one return push_data['before'] if parents.empty? @@ -112,35 +112,36 @@ class IrkerWorker # rubocop:disable Scalability/IdempotentWorker parents[0].id end - def send_commits_count(data, project, repo, committer, branch) - url = compare_url data, project.full_path - commits = colorize_commits data['total_commits_count'] + def send_commits_count(push_data) + url = compare_url(push_data['before'], push_data['after']) + commits = colorize_commits(push_data['total_commits_count']) + new_commits = 'new commit'.pluralize(push_data['total_commits_count']) - new_commits = 'new commit'.pluralize(data['total_commits_count']) - sendtoirker "[#{repo}] #{committer} pushed #{commits} #{new_commits} " \ - "to #{branch}: #{url}" + send_to_irker("[#{@repo_name}] #{@committer} pushed #{commits} #{new_commits} " \ + "to #{@branch}: #{url}") end - def compare_url(data, repo_path) - sha1 = Commit.truncate_sha(data['before']) - sha2 = Commit.truncate_sha(data['after']) - compare_url = "#{Gitlab.config.gitlab.url}/#{repo_path}/-/compare" \ + def compare_url(sha_before, sha_after) + sha1 = Commit.truncate_sha(sha_before) + sha2 = Commit.truncate_sha(sha_after) + compare_url = "#{Gitlab.config.gitlab.url}/#{@repo_path}/-/compare" \ "/#{sha1}...#{sha2}" - colorize_url compare_url + + colorize_url(compare_url) end - def send_one_commit(project, hook_attrs, repo_name, branch) - commit = commit_from_id project, hook_attrs['id'] - sha = colorize_sha Commit.truncate_sha(hook_attrs['id']) - author = hook_attrs['author']['name'] + def send_one_commit(commit_attrs) + commit = commit_from_id(commit_attrs['id']) + sha = colorize_sha(Commit.truncate_sha(commit_attrs['id'])) + author = commit_attrs['author']['name'] files = colorize_nb_files(files_count(commit)) title = commit.title - sendtoirker "#{repo_name}/#{branch} #{sha} #{author} (#{files}): #{title}" + send_to_irker("#{@repo_name}/#{@branch} #{sha} #{author} (#{files}): #{title}") end - def commit_from_id(project, id) - project.commit(id) + def commit_from_id(id) + @project.commit(id) end def files_count(commit) diff --git a/app/workers/mail_scheduler/notification_service_worker.rb b/app/workers/mail_scheduler/notification_service_worker.rb index 691af2a724d..309f23c8708 100644 --- a/app/workers/mail_scheduler/notification_service_worker.rb +++ b/app/workers/mail_scheduler/notification_service_worker.rb @@ -9,6 +9,7 @@ module MailScheduler feature_category :issue_tracking worker_resource_boundary :cpu + loggable_arguments 0 def perform(meth, *args) check_arguments!(args) diff --git a/app/workers/merge_worker.rb b/app/workers/merge_worker.rb index cc5fe884aec..270bd831f96 100644 --- a/app/workers/merge_worker.rb +++ b/app/workers/merge_worker.rb @@ -6,6 +6,7 @@ class MergeWorker # rubocop:disable Scalability/IdempotentWorker feature_category :source_code_management urgency :high weight 5 + loggable_arguments 2 def perform(merge_request_id, current_user_id, params) params = params.with_indifferent_access diff --git a/app/workers/metrics/dashboard/prune_old_annotations_worker.rb b/app/workers/metrics/dashboard/prune_old_annotations_worker.rb new file mode 100644 index 00000000000..2a9ce3bb8e6 --- /dev/null +++ b/app/workers/metrics/dashboard/prune_old_annotations_worker.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module Metrics + module Dashboard + class PruneOldAnnotationsWorker + include ApplicationWorker + + DELETE_LIMIT = 10_000 + DEFAULT_CUT_OFF_PERIOD = 2.weeks + + feature_category :metrics + + idempotent! # in the scope of 24 hours + + def perform + stale_annotations = ::Metrics::Dashboard::Annotation.ending_before(DEFAULT_CUT_OFF_PERIOD.ago.beginning_of_day) + stale_annotations.delete_with_limit(DELETE_LIMIT) + + self.class.perform_async if stale_annotations.exists? + end + end + end +end diff --git a/app/workers/metrics/dashboard/schedule_annotations_prune_worker.rb b/app/workers/metrics/dashboard/schedule_annotations_prune_worker.rb new file mode 100644 index 00000000000..cbdd69c6e8c --- /dev/null +++ b/app/workers/metrics/dashboard/schedule_annotations_prune_worker.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module Metrics + module Dashboard + class ScheduleAnnotationsPruneWorker + include ApplicationWorker + # rubocop:disable Scalability/CronWorkerContext + # This worker does not perform work scoped to a context + include CronjobQueue + # rubocop:enable Scalability/CronWorkerContext + + feature_category :metrics + + idempotent! # PruneOldAnnotationsWorker worker is idempotent in the scope of 24 hours + + def perform + # Process is split into two jobs to avoid long running jobs, which are more prone to be disrupted + # mid work, which may cause some data not be delete, especially because cronjobs has retry option disabled + PruneOldAnnotationsWorker.perform_async + end + end + end +end diff --git a/app/workers/new_note_worker.rb b/app/workers/new_note_worker.rb index 8ead87a9230..b31311b0e44 100644 --- a/app/workers/new_note_worker.rb +++ b/app/workers/new_note_worker.rb @@ -16,17 +16,14 @@ class NewNoteWorker # rubocop:disable Scalability/IdempotentWorker NotificationService.new.new_note(note) unless skip_notification?(note) Notes::PostProcessService.new(note).execute else - Rails.logger.error("NewNoteWorker: couldn't find note with ID=#{note_id}, skipping job") # rubocop:disable Gitlab/RailsLogger + Gitlab::AppLogger.error("NewNoteWorker: couldn't find note with ID=#{note_id}, skipping job") end end + # rubocop: enable CodeReuse/ActiveRecord private - # EE-only method def skip_notification?(note) - false + note.review.present? end - # rubocop: enable CodeReuse/ActiveRecord end - -NewNoteWorker.prepend_if_ee('EE::NewNoteWorker') diff --git a/app/workers/object_storage/background_move_worker.rb b/app/workers/object_storage/background_move_worker.rb index 7b0a7c7ec58..fba91e49e43 100644 --- a/app/workers/object_storage/background_move_worker.rb +++ b/app/workers/object_storage/background_move_worker.rb @@ -7,6 +7,7 @@ module ObjectStorage sidekiq_options retry: 5 feature_category_not_owned! + loggable_arguments 0, 1, 2, 3 def perform(uploader_class_name, subject_class_name, file_field, subject_id) uploader_class = uploader_class_name.constantize diff --git a/app/workers/object_storage/migrate_uploads_worker.rb b/app/workers/object_storage/migrate_uploads_worker.rb index d9d21f2cb7e..33a0e0f5f88 100644 --- a/app/workers/object_storage/migrate_uploads_worker.rb +++ b/app/workers/object_storage/migrate_uploads_worker.rb @@ -7,6 +7,7 @@ module ObjectStorage include ObjectStorageQueue feature_category_not_owned! + loggable_arguments 0, 1, 2, 3 SanityCheckError = Class.new(StandardError) diff --git a/app/workers/pages_worker.rb b/app/workers/pages_worker.rb index 875f17282f9..d699e32c1a0 100644 --- a/app/workers/pages_worker.rb +++ b/app/workers/pages_worker.rb @@ -5,6 +5,7 @@ class PagesWorker # rubocop:disable Scalability/IdempotentWorker sidekiq_options retry: 3 feature_category :pages + loggable_arguments 0, 1 def perform(action, *arg) send(action, *arg) # rubocop:disable GitlabSecurity/PublicSend diff --git a/app/workers/personal_access_tokens/expiring_worker.rb b/app/workers/personal_access_tokens/expiring_worker.rb index 86240f827fc..f9940d9d014 100644 --- a/app/workers/personal_access_tokens/expiring_worker.rb +++ b/app/workers/personal_access_tokens/expiring_worker.rb @@ -15,9 +15,9 @@ module PersonalAccessTokens with_context(user: user) do notification_service.access_token_about_to_expire(user) - Rails.logger.info "#{self.class}: Notifying User #{user.id} about expiring tokens" # rubocop:disable Gitlab/RailsLogger + Gitlab::AppLogger.info "#{self.class}: Notifying User #{user.id} about expiring tokens" - user.personal_access_tokens.expiring_and_not_notified(limit_date).update_all(expire_notification_delivered: true) + user.personal_access_tokens.without_impersonation.expiring_and_not_notified(limit_date).update_all(expire_notification_delivered: true) end end end diff --git a/app/workers/pipeline_notification_worker.rb b/app/workers/pipeline_notification_worker.rb index 3336383adf7..f4b43106bf2 100644 --- a/app/workers/pipeline_notification_worker.rb +++ b/app/workers/pipeline_notification_worker.rb @@ -7,10 +7,10 @@ class PipelineNotificationWorker # rubocop:disable Scalability/IdempotentWorker urgency :high worker_resource_boundary :cpu - # rubocop: disable CodeReuse/ActiveRecord def perform(pipeline_id, args = {}) case args when Hash + args = args.with_indifferent_access ref_status = args[:ref_status] recipients = args[:recipients] else # TODO: backward compatible interface, can be removed in 12.10 @@ -18,10 +18,9 @@ class PipelineNotificationWorker # rubocop:disable Scalability/IdempotentWorker ref_status = nil end - pipeline = Ci::Pipeline.find_by(id: pipeline_id) + pipeline = Ci::Pipeline.find_by_id(pipeline_id) return unless pipeline NotificationService.new.pipeline_finished(pipeline, ref_status: ref_status, recipients: recipients) end - # rubocop: enable CodeReuse/ActiveRecord end diff --git a/app/workers/pipeline_process_worker.rb b/app/workers/pipeline_process_worker.rb index 66a661dde71..cd7c82d3117 100644 --- a/app/workers/pipeline_process_worker.rb +++ b/app/workers/pipeline_process_worker.rb @@ -7,6 +7,7 @@ class PipelineProcessWorker # rubocop:disable Scalability/IdempotentWorker queue_namespace :pipeline_processing feature_category :continuous_integration urgency :high + loggable_arguments 1 # rubocop: disable CodeReuse/ActiveRecord def perform(pipeline_id, build_ids = nil) diff --git a/app/workers/pipeline_update_ci_ref_status_worker.rb b/app/workers/pipeline_update_ci_ref_status_worker.rb index 96e14e126de..9b1a5d8e7cf 100644 --- a/app/workers/pipeline_update_ci_ref_status_worker.rb +++ b/app/workers/pipeline_update_ci_ref_status_worker.rb @@ -1,5 +1,6 @@ # frozen_string_literal: true +# NOTE: This class is unused and to be removed in 13.1~ class PipelineUpdateCiRefStatusWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker include PipelineQueue diff --git a/app/workers/post_receive.rb b/app/workers/post_receive.rb index ddf112e648c..62d76294bc0 100644 --- a/app/workers/post_receive.rb +++ b/app/workers/post_receive.rb @@ -7,6 +7,7 @@ class PostReceive # rubocop:disable Scalability/IdempotentWorker urgency :high worker_resource_boundary :cpu weight 5 + loggable_arguments 0, 1, 2, 3 def perform(gl_repository, identifier, changes, push_options = {}) container, project, repo_type = Gitlab::GlRepository.parse(gl_repository) diff --git a/app/workers/process_commit_worker.rb b/app/workers/process_commit_worker.rb index bdfabea8938..5756ebb8358 100644 --- a/app/workers/process_commit_worker.rb +++ b/app/workers/process_commit_worker.rb @@ -13,8 +13,8 @@ class ProcessCommitWorker feature_category :source_code_management urgency :high weight 3 - idempotent! + loggable_arguments 2, 3 # project_id - The ID of the project this commit belongs to. # user_id - The ID of the user that pushed the commit. diff --git a/app/workers/project_cache_worker.rb b/app/workers/project_cache_worker.rb index 573f903f4e0..b114c67de47 100644 --- a/app/workers/project_cache_worker.rb +++ b/app/workers/project_cache_worker.rb @@ -4,11 +4,11 @@ class ProjectCacheWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker - urgency :high - LEASE_TIMEOUT = 15.minutes.to_i feature_category :source_code_management + urgency :high + loggable_arguments 1, 2, 3 # project_id - The ID of the project for which to flush the cache. # files - An Array containing extra types of files to refresh such as diff --git a/app/workers/project_export_worker.rb b/app/workers/project_export_worker.rb index a287c511a65..d29348e85bc 100644 --- a/app/workers/project_export_worker.rb +++ b/app/workers/project_export_worker.rb @@ -8,6 +8,7 @@ class ProjectExportWorker # rubocop:disable Scalability/IdempotentWorker feature_category :importers worker_resource_boundary :memory urgency :throttled + loggable_arguments 2, 3 def perform(current_user_id, project_id, after_export_strategy = {}, params = {}) current_user = User.find(current_user_id) diff --git a/app/workers/propagate_integration_worker.rb b/app/workers/propagate_integration_worker.rb new file mode 100644 index 00000000000..15c0e761a0a --- /dev/null +++ b/app/workers/propagate_integration_worker.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +class PropagateIntegrationWorker + include ApplicationWorker + + feature_category :integrations + idempotent! + loggable_arguments 1 + + def perform(integration_id, overwrite) + Admin::PropagateIntegrationService.propagate( + integration: Service.find(integration_id), + overwrite: overwrite + ) + end +end diff --git a/app/workers/rebase_worker.rb b/app/workers/rebase_worker.rb index 2e13af5e0aa..ee9ae827bb6 100644 --- a/app/workers/rebase_worker.rb +++ b/app/workers/rebase_worker.rb @@ -7,6 +7,7 @@ class RebaseWorker # rubocop:disable Scalability/IdempotentWorker feature_category :source_code_management weight 2 + loggable_arguments 2 def perform(merge_request_id, current_user_id, skip_ci = false) current_user = User.find(current_user_id) diff --git a/app/workers/remove_expired_group_links_worker.rb b/app/workers/remove_expired_group_links_worker.rb index 8226f22837c..3f1a484f384 100644 --- a/app/workers/remove_expired_group_links_worker.rb +++ b/app/workers/remove_expired_group_links_worker.rb @@ -7,10 +7,12 @@ class RemoveExpiredGroupLinksWorker # rubocop:disable Scalability/IdempotentWork feature_category :authentication_and_authorization def perform - ProjectGroupLink.expired.destroy_all # rubocop: disable DestroyAll + ProjectGroupLink.expired.find_each do |link| + Projects::GroupLinks::DestroyService.new(link.project, nil).execute(link) + end GroupGroupLink.expired.find_in_batches do |link_batch| - Groups::GroupLinks::DestroyService.new(nil, nil).execute(link_batch) + Groups::GroupLinks::DestroyService.new(nil, nil).execute(link_batch, skip_authorization: true) end end end diff --git a/app/workers/repository_check/batch_worker.rb b/app/workers/repository_check/batch_worker.rb index 9e762860ec6..1e2cb912598 100644 --- a/app/workers/repository_check/batch_worker.rb +++ b/app/workers/repository_check/batch_worker.rb @@ -12,6 +12,8 @@ module RepositoryCheck attr_reader :shard_name + loggable_arguments 0 + def perform(shard_name) @shard_name = shard_name diff --git a/app/workers/repository_import_worker.rb b/app/workers/repository_import_worker.rb index 9f17ef467e3..30570a2227e 100644 --- a/app/workers/repository_import_worker.rb +++ b/app/workers/repository_import_worker.rb @@ -43,7 +43,12 @@ class RepositoryImportWorker # rubocop:disable Scalability/IdempotentWorker def start_import return true if start(project.import_state) - Rails.logger.info("Project #{project.full_path} was in inconsistent state (#{project.import_status}) while importing.") # rubocop:disable Gitlab/RailsLogger + Gitlab::Import::Logger.info( + message: 'Project was in inconsistent state while importing', + project_full_path: project.full_path, + project_import_status: project.import_status + ) + false end diff --git a/app/workers/repository_remove_remote_worker.rb b/app/workers/repository_remove_remote_worker.rb index 23a9ec1e202..5e632b1b1ca 100644 --- a/app/workers/repository_remove_remote_worker.rb +++ b/app/workers/repository_remove_remote_worker.rb @@ -5,6 +5,7 @@ class RepositoryRemoveRemoteWorker # rubocop:disable Scalability/IdempotentWorke include ExclusiveLeaseGuard feature_category :source_code_management + loggable_arguments 1 LEASE_TIMEOUT = 1.hour diff --git a/app/workers/repository_update_remote_mirror_worker.rb b/app/workers/repository_update_remote_mirror_worker.rb index cfff2382f04..21b5916f459 100644 --- a/app/workers/repository_update_remote_mirror_worker.rb +++ b/app/workers/repository_update_remote_mirror_worker.rb @@ -10,6 +10,7 @@ class RepositoryUpdateRemoteMirrorWorker # rubocop:disable Scalability/Idempoten sidekiq_options retry: 3, dead: false feature_category :source_code_management + loggable_arguments 1 LOCK_WAIT_TIME = 30.seconds MAX_TRIES = 3 diff --git a/app/workers/stuck_import_jobs_worker.rb b/app/workers/stuck_import_jobs_worker.rb index 6a48b78b22c..ce8d5bf0219 100644 --- a/app/workers/stuck_import_jobs_worker.rb +++ b/app/workers/stuck_import_jobs_worker.rb @@ -1,75 +1,19 @@ # frozen_string_literal: true class StuckImportJobsWorker # rubocop:disable Scalability/IdempotentWorker - include ApplicationWorker - # rubocop:disable Scalability/CronWorkerContext - # This worker updates several import states inline and does not schedule - # other jobs. So no context needed - include CronjobQueue - # rubocop:enable Scalability/CronWorkerContext - - feature_category :importers - worker_resource_boundary :cpu - - IMPORT_JOBS_EXPIRATION = 15.hours.to_i - - def perform - import_state_without_jid_count = mark_import_states_without_jid_as_failed! - import_state_with_jid_count = mark_import_states_with_jid_as_failed! - - Gitlab::Metrics.add_event(:stuck_import_jobs, - projects_without_jid_count: import_state_without_jid_count, - projects_with_jid_count: import_state_with_jid_count) - end + include Gitlab::Import::StuckImportJob private - def mark_import_states_without_jid_as_failed! - enqueued_import_states_without_jid.each do |import_state| - import_state.mark_as_failed(error_message) - end.count - end - - # rubocop: disable CodeReuse/ActiveRecord - def mark_import_states_with_jid_as_failed! - jids_and_ids = enqueued_import_states_with_jid.pluck(:jid, :id).to_h - - # Find the jobs that aren't currently running or that exceeded the threshold. - completed_jids = Gitlab::SidekiqStatus.completed_jids(jids_and_ids.keys) - return unless completed_jids.any? - - completed_import_state_ids = jids_and_ids.values_at(*completed_jids) - - # We select the import states again, because they may have transitioned from - # scheduled/started to finished/failed while we were looking up their Sidekiq status. - completed_import_states = enqueued_import_states_with_jid.where(id: completed_import_state_ids) - - completed_import_state_jids = completed_import_states.map { |import_state| import_state.jid }.join(', ') - Rails.logger.info("Marked stuck import jobs as failed. JIDs: #{completed_import_state_jids}") # rubocop:disable Gitlab/RailsLogger - - completed_import_states.each do |import_state| - import_state.mark_as_failed(error_message) - end.count + def track_metrics(with_jid_count, without_jid_count) + Gitlab::Metrics.add_event( + :stuck_import_jobs, + projects_without_jid_count: without_jid_count, + projects_with_jid_count: with_jid_count + ) end - # rubocop: enable CodeReuse/ActiveRecord def enqueued_import_states ProjectImportState.with_status([:scheduled, :started]) end - - # rubocop: disable CodeReuse/ActiveRecord - def enqueued_import_states_with_jid - enqueued_import_states.where.not(jid: nil) - end - # rubocop: enable CodeReuse/ActiveRecord - - # rubocop: disable CodeReuse/ActiveRecord - def enqueued_import_states_without_jid - enqueued_import_states.where(jid: nil) - end - # rubocop: enable CodeReuse/ActiveRecord - - def error_message - _("Import timed out. Import took longer than %{import_jobs_expiration} seconds") % { import_jobs_expiration: IMPORT_JOBS_EXPIRATION } - end end diff --git a/app/workers/todos_destroyer/entity_leave_worker.rb b/app/workers/todos_destroyer/entity_leave_worker.rb index 558cc32d158..4996456dc91 100644 --- a/app/workers/todos_destroyer/entity_leave_worker.rb +++ b/app/workers/todos_destroyer/entity_leave_worker.rb @@ -5,6 +5,8 @@ module TodosDestroyer include ApplicationWorker include TodosDestroyerQueue + loggable_arguments 2 + def perform(user_id, entity_id, entity_type) ::Todos::Destroy::EntityLeaveService.new(user_id, entity_id, entity_type).execute end diff --git a/app/workers/update_external_pull_requests_worker.rb b/app/workers/update_external_pull_requests_worker.rb index 0d48877e1b0..e916331ae82 100644 --- a/app/workers/update_external_pull_requests_worker.rb +++ b/app/workers/update_external_pull_requests_worker.rb @@ -5,6 +5,7 @@ class UpdateExternalPullRequestsWorker # rubocop:disable Scalability/IdempotentW feature_category :source_code_management weight 3 + loggable_arguments 2 def perform(project_id, user_id, ref) project = Project.find_by_id(project_id) diff --git a/app/workers/update_merge_requests_worker.rb b/app/workers/update_merge_requests_worker.rb index 63bb6171b9c..98534b258a7 100644 --- a/app/workers/update_merge_requests_worker.rb +++ b/app/workers/update_merge_requests_worker.rb @@ -7,6 +7,7 @@ class UpdateMergeRequestsWorker # rubocop:disable Scalability/IdempotentWorker urgency :high worker_resource_boundary :cpu weight 3 + loggable_arguments 2, 3, 4 LOG_TIME_THRESHOLD = 90 # seconds diff --git a/app/workers/web_hook_worker.rb b/app/workers/web_hook_worker.rb index 6e1e7e7d62e..5230f3bfa1f 100644 --- a/app/workers/web_hook_worker.rb +++ b/app/workers/web_hook_worker.rb @@ -5,6 +5,7 @@ class WebHookWorker # rubocop:disable Scalability/IdempotentWorker feature_category :integrations worker_has_external_dependencies! + loggable_arguments 2 sidekiq_options retry: 4, dead: false |