diff options
Diffstat (limited to 'app/workers')
54 files changed, 564 insertions, 88 deletions
diff --git a/app/workers/all_queues.yml b/app/workers/all_queues.yml index 6f080a97f7a..1f2e8213b64 100644 --- a/app/workers/all_queues.yml +++ b/app/workers/all_queues.yml @@ -124,7 +124,7 @@ :idempotent: :tags: [] - :name: cronjob:analytics_instance_statistics_count_job_trigger - :feature_category: :instance_statistics + :feature_category: :devops_reports :has_external_dependencies: :urgency: :low :resource_boundary: :unknown @@ -323,6 +323,22 @@ :weight: 1 :idempotent: :tags: [] +- :name: cronjob:releases_create_evidence + :feature_category: :release_evidence + :has_external_dependencies: + :urgency: :low + :resource_boundary: :unknown + :weight: 1 + :idempotent: + :tags: [] +- :name: cronjob:releases_manage_evidence + :feature_category: :release_evidence + :has_external_dependencies: + :urgency: :low + :resource_boundary: :unknown + :weight: 1 + :idempotent: + :tags: [] - :name: cronjob:remove_expired_group_links :feature_category: :authentication_and_authorization :has_external_dependencies: @@ -380,7 +396,7 @@ :idempotent: :tags: [] - :name: cronjob:schedule_merge_request_cleanup_refs - :feature_category: :source_code_management + :feature_category: :code_review :has_external_dependencies: :urgency: :low :resource_boundary: :unknown @@ -388,7 +404,7 @@ :idempotent: true :tags: [] - :name: cronjob:schedule_migrate_external_diffs - :feature_category: :source_code_management + :feature_category: :code_review :has_external_dependencies: :urgency: :low :resource_boundary: :unknown @@ -412,7 +428,7 @@ :idempotent: :tags: [] - :name: cronjob:stuck_merge_jobs - :feature_category: :source_code_management + :feature_category: :code_review :has_external_dependencies: :urgency: :low :resource_boundary: :unknown @@ -691,6 +707,22 @@ :weight: 1 :idempotent: :tags: [] +- :name: github_importer:github_import_import_pull_request_merged_by + :feature_category: :importers + :has_external_dependencies: true + :urgency: :low + :resource_boundary: :unknown + :weight: 1 + :idempotent: + :tags: [] +- :name: github_importer:github_import_import_pull_request_review + :feature_category: :importers + :has_external_dependencies: true + :urgency: :low + :resource_boundary: :unknown + :weight: 1 + :idempotent: + :tags: [] - :name: github_importer:github_import_refresh_import_jid :feature_category: :importers :has_external_dependencies: @@ -747,6 +779,22 @@ :weight: 1 :idempotent: :tags: [] +- :name: github_importer:github_import_stage_import_pull_requests_merged_by + :feature_category: :importers + :has_external_dependencies: + :urgency: :low + :resource_boundary: :unknown + :weight: 1 + :idempotent: + :tags: [] +- :name: github_importer:github_import_stage_import_pull_requests_reviews + :feature_category: :importers + :has_external_dependencies: + :urgency: :low + :resource_boundary: :unknown + :weight: 1 + :idempotent: + :tags: [] - :name: github_importer:github_import_stage_import_repository :feature_category: :importers :has_external_dependencies: @@ -829,15 +877,23 @@ :tags: [] - :name: jira_connect:jira_connect_sync_branch :feature_category: :integrations - :has_external_dependencies: + :has_external_dependencies: true :urgency: :low :resource_boundary: :unknown :weight: 1 :idempotent: :tags: [] +- :name: jira_connect:jira_connect_sync_builds + :feature_category: :integrations + :has_external_dependencies: true + :urgency: :low + :resource_boundary: :unknown + :weight: 1 + :idempotent: true + :tags: [] - :name: jira_connect:jira_connect_sync_merge_request :feature_category: :integrations - :has_external_dependencies: + :has_external_dependencies: true :urgency: :low :resource_boundary: :unknown :weight: 1 @@ -1045,6 +1101,14 @@ :weight: 1 :idempotent: true :tags: [] +- :name: pipeline_background:ci_test_failure_history + :feature_category: :continuous_integration + :has_external_dependencies: + :urgency: :low + :resource_boundary: :unknown + :weight: 1 + :idempotent: true + :tags: [] - :name: pipeline_cache:expire_job_cache :feature_category: :continuous_integration :has_external_dependencies: @@ -1313,7 +1377,15 @@ :idempotent: true :tags: [] - :name: analytics_instance_statistics_counter_job - :feature_category: :instance_statistics + :feature_category: :devops_reports + :has_external_dependencies: + :urgency: :low + :resource_boundary: :unknown + :weight: 1 + :idempotent: true + :tags: [] +- :name: approve_blocked_pending_approval_users + :feature_category: :users :has_external_dependencies: :urgency: :low :resource_boundary: :unknown @@ -1377,16 +1449,8 @@ :weight: 2 :idempotent: true :tags: [] -- :name: create_evidence - :feature_category: :release_evidence - :has_external_dependencies: - :urgency: :low - :resource_boundary: :unknown - :weight: 2 - :idempotent: - :tags: [] - :name: create_note_diff_file - :feature_category: :source_code_management + :feature_category: :code_review :has_external_dependencies: :urgency: :low :resource_boundary: :unknown @@ -1402,7 +1466,7 @@ :idempotent: :tags: [] - :name: delete_diff_files - :feature_category: :source_code_management + :feature_category: :code_review :has_external_dependencies: :urgency: :low :resource_boundary: :unknown @@ -1497,6 +1561,14 @@ :weight: 2 :idempotent: :tags: [] +- :name: environments_canary_ingress_update + :feature_category: :continuous_delivery + :has_external_dependencies: true + :urgency: :low + :resource_boundary: :unknown + :weight: 1 + :idempotent: true + :tags: [] - :name: error_tracking_issue_link :feature_category: :error_tracking :has_external_dependencies: true @@ -1562,6 +1634,14 @@ :weight: 1 :idempotent: :tags: [] +- :name: gitlab_performance_bar_stats + :feature_category: :metrics + :has_external_dependencies: + :urgency: :low + :resource_boundary: :unknown + :weight: 1 + :idempotent: true + :tags: [] - :name: gitlab_shell :feature_category: :source_code_management :has_external_dependencies: @@ -1601,7 +1681,7 @@ :urgency: :low :resource_boundary: :cpu :weight: 2 - :idempotent: + :idempotent: true :tags: [] - :name: invalid_gpg_signature_update :feature_category: :source_code_management @@ -1660,7 +1740,7 @@ :idempotent: :tags: [] - :name: merge_request_cleanup_refs - :feature_category: :source_code_management + :feature_category: :code_review :has_external_dependencies: :urgency: :low :resource_boundary: :unknown @@ -1668,7 +1748,7 @@ :idempotent: true :tags: [] - :name: merge_request_mergeability_check - :feature_category: :source_code_management + :feature_category: :code_review :has_external_dependencies: :urgency: :low :resource_boundary: :unknown @@ -1692,7 +1772,7 @@ :idempotent: true :tags: [] - :name: migrate_external_diffs - :feature_category: :source_code_management + :feature_category: :code_review :has_external_dependencies: :urgency: :low :resource_boundary: :unknown @@ -1707,6 +1787,14 @@ :weight: 1 :idempotent: :tags: [] +- :name: namespaces_onboarding_user_added + :feature_category: :users + :has_external_dependencies: + :urgency: :low + :resource_boundary: :unknown + :weight: 1 + :idempotent: true + :tags: [] - :name: new_issue :feature_category: :issue_tracking :has_external_dependencies: @@ -1716,7 +1804,7 @@ :idempotent: :tags: [] - :name: new_merge_request - :feature_category: :source_code_management + :feature_category: :code_review :has_external_dependencies: :urgency: :high :resource_boundary: :cpu @@ -1812,7 +1900,7 @@ :urgency: :high :resource_boundary: :unknown :weight: 1 - :idempotent: + :idempotent: true :tags: [] - :name: project_daily_statistics :feature_category: :source_code_management @@ -1839,6 +1927,14 @@ :weight: 1 :idempotent: :tags: [] +- :name: project_schedule_bulk_repository_shard_moves + :feature_category: :gitaly + :has_external_dependencies: + :urgency: :throttled + :resource_boundary: :unknown + :weight: 1 + :idempotent: true + :tags: [] - :name: project_service :feature_category: :integrations :has_external_dependencies: true @@ -1973,7 +2069,7 @@ :urgency: :low :resource_boundary: :unknown :weight: 1 - :idempotent: + :idempotent: true :tags: [] - :name: self_monitoring_project_create :feature_category: :metrics @@ -2024,7 +2120,7 @@ :idempotent: true :tags: [] - :name: update_merge_requests - :feature_category: :source_code_management + :feature_category: :code_review :has_external_dependencies: :urgency: :high :resource_boundary: :cpu diff --git a/app/workers/analytics/instance_statistics/count_job_trigger_worker.rb b/app/workers/analytics/instance_statistics/count_job_trigger_worker.rb index bf57619fc6e..81a765d5d08 100644 --- a/app/workers/analytics/instance_statistics/count_job_trigger_worker.rb +++ b/app/workers/analytics/instance_statistics/count_job_trigger_worker.rb @@ -8,7 +8,7 @@ module Analytics DEFAULT_DELAY = 3.minutes.freeze - feature_category :instance_statistics + feature_category :devops_reports urgency :low idempotent! diff --git a/app/workers/analytics/instance_statistics/counter_job_worker.rb b/app/workers/analytics/instance_statistics/counter_job_worker.rb index 7fc715419b8..c07b2569453 100644 --- a/app/workers/analytics/instance_statistics/counter_job_worker.rb +++ b/app/workers/analytics/instance_statistics/counter_job_worker.rb @@ -5,7 +5,7 @@ module Analytics class CounterJobWorker include ApplicationWorker - feature_category :instance_statistics + feature_category :devops_reports urgency :low idempotent! diff --git a/app/workers/approve_blocked_pending_approval_users_worker.rb b/app/workers/approve_blocked_pending_approval_users_worker.rb new file mode 100644 index 00000000000..8ca61d68bfd --- /dev/null +++ b/app/workers/approve_blocked_pending_approval_users_worker.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +class ApproveBlockedPendingApprovalUsersWorker + include ApplicationWorker + + idempotent! + + feature_category :users + + def perform(current_user_id) + current_user = User.find(current_user_id) + + User.blocked_pending_approval.find_each do |user| + Users::ApproveService.new(current_user).execute(user) + end + end +end diff --git a/app/workers/build_finished_worker.rb b/app/workers/build_finished_worker.rb index af2305528ce..d7a5fcf4f18 100644 --- a/app/workers/build_finished_worker.rb +++ b/app/workers/build_finished_worker.rb @@ -33,11 +33,6 @@ class BuildFinishedWorker # rubocop:disable Scalability/IdempotentWorker BuildCoverageWorker.new.perform(build.id) Ci::BuildReportResultWorker.new.perform(build.id) - # TODO: As per https://gitlab.com/groups/gitlab-com/gl-infra/-/epics/194, it may be - # best to avoid creating more workers that we have no intention of calling async. - # Change the previous worker calls on top to also just call the service directly. - Ci::TestCasesService.new.execute(build) - # We execute these async as these are independent operations. BuildHooksWorker.perform_async(build.id) ExpirePipelineCacheWorker.perform_async(build.pipeline_id) if build.pipeline.cacheable? diff --git a/app/workers/ci/test_failure_history_worker.rb b/app/workers/ci/test_failure_history_worker.rb new file mode 100644 index 00000000000..e1562cb3836 --- /dev/null +++ b/app/workers/ci/test_failure_history_worker.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +module Ci + class TestFailureHistoryWorker + include ApplicationWorker + include PipelineBackgroundQueue + + idempotent! + + def perform(pipeline_id) + Ci::Pipeline.find_by_id(pipeline_id).try do |pipeline| + Ci::TestFailureHistoryService.new(pipeline).execute + end + end + end +end diff --git a/app/workers/clusters/applications/check_prometheus_health_worker.rb b/app/workers/clusters/applications/check_prometheus_health_worker.rb index 2e8ee739946..cf9534c9a78 100644 --- a/app/workers/clusters/applications/check_prometheus_health_worker.rb +++ b/app/workers/clusters/applications/check_prometheus_health_worker.rb @@ -20,7 +20,7 @@ module Clusters demo_project_ids = Gitlab::Monitor::DemoProjects.primary_keys clusters = Clusters::Cluster.with_application_prometheus - .with_project_alert_service_data(demo_project_ids) + .with_project_http_integrations(demo_project_ids) # Move to a seperate worker with scoped context if expanded to do work on customer projects clusters.each { |cluster| Clusters::Applications::PrometheusHealthCheckService.new(cluster).execute } diff --git a/app/workers/concerns/gitlab/github_import/object_importer.rb b/app/workers/concerns/gitlab/github_import/object_importer.rb index 63c1ba8e699..575cd4862b0 100644 --- a/app/workers/concerns/gitlab/github_import/object_importer.rb +++ b/app/workers/concerns/gitlab/github_import/object_importer.rb @@ -15,17 +15,25 @@ module Gitlab feature_category :importers worker_has_external_dependencies! + + def logger + @logger ||= Gitlab::Import::Logger.build + end end # project - An instance of `Project` to import the data into. # client - An instance of `Gitlab::GithubImport::Client` # hash - A Hash containing the details of the object to import. def import(project, client, hash) - object = representation_class.from_json_hash(hash) + info(project.id, message: 'starting importer') + object = representation_class.from_json_hash(hash) importer_class.new(object, project, client).execute counter.increment + info(project.id, message: 'importer finished') + rescue => e + error(project.id, e) end def counter @@ -52,6 +60,35 @@ module Gitlab def counter_description raise NotImplementedError end + + private + + def info(project_id, extra = {}) + logger.info(log_attributes(project_id, extra)) + end + + def error(project_id, exception) + logger.error( + log_attributes( + project_id, + message: 'importer failed', + 'error.message': exception.message + ) + ) + + Gitlab::ErrorTracking.track_and_raise_exception( + exception, + log_attributes(project_id) + ) + end + + def log_attributes(project_id, extra = {}) + extra.merge( + import_source: :github, + project_id: project_id, + importer: importer_class.name + ) + end end end end diff --git a/app/workers/concerns/gitlab/github_import/rescheduling_methods.rb b/app/workers/concerns/gitlab/github_import/rescheduling_methods.rb index 1c6413674a0..eb1af0869bd 100644 --- a/app/workers/concerns/gitlab/github_import/rescheduling_methods.rb +++ b/app/workers/concerns/gitlab/github_import/rescheduling_methods.rb @@ -6,7 +6,7 @@ module Gitlab # importing GitHub projects. module ReschedulingMethods # project_id - The ID of the GitLab project to import the note into. - # hash - A Hash containing the details of the GitHub object to imoprt. + # hash - A Hash containing the details of the GitHub object to import. # notify_key - The Redis key to notify upon completion, if any. # rubocop: disable CodeReuse/ActiveRecord def perform(project_id, hash, notify_key = nil) diff --git a/app/workers/concerns/gitlab/github_import/stage_methods.rb b/app/workers/concerns/gitlab/github_import/stage_methods.rb index e2dee315cde..e5985fb94da 100644 --- a/app/workers/concerns/gitlab/github_import/stage_methods.rb +++ b/app/workers/concerns/gitlab/github_import/stage_methods.rb @@ -5,11 +5,17 @@ module Gitlab module StageMethods # project_id - The ID of the GitLab project to import the data into. def perform(project_id) + info(project_id, message: 'starting stage') + return unless (project = find_project(project_id)) client = GithubImport.new_client_for(project) try_import(client, project) + + info(project_id, message: 'stage finished') + rescue => e + error(project_id, e) end # client - An instance of Gitlab::GithubImport::Client. @@ -27,6 +33,39 @@ module Gitlab Project.joins_import_state.where(import_state: { status: :started }).find_by(id: id) end # rubocop: enable CodeReuse/ActiveRecord + + private + + def info(project_id, extra = {}) + logger.info(log_attributes(project_id, extra)) + end + + def error(project_id, exception) + logger.error( + log_attributes( + project_id, + message: 'stage failed', + 'error.message': exception.message + ) + ) + + Gitlab::ErrorTracking.track_and_raise_exception( + exception, + log_attributes(project_id) + ) + end + + def log_attributes(project_id, extra = {}) + extra.merge( + import_source: :github, + project_id: project_id, + import_stage: self.class.name + ) + end + + def logger + @logger ||= Gitlab::Import::Logger.build + end end end end diff --git a/app/workers/concerns/limited_capacity/worker.rb b/app/workers/concerns/limited_capacity/worker.rb index b5a97e49300..9dd8d942146 100644 --- a/app/workers/concerns/limited_capacity/worker.rb +++ b/app/workers/concerns/limited_capacity/worker.rb @@ -73,7 +73,7 @@ module LimitedCapacity raise ensure job_tracker.remove(jid) - report_prometheus_metrics + report_prometheus_metrics(*args) re_enqueue(*args) unless exception end diff --git a/app/workers/concerns/reenqueuer.rb b/app/workers/concerns/reenqueuer.rb index 6f399b6d90b..641ca691868 100644 --- a/app/workers/concerns/reenqueuer.rb +++ b/app/workers/concerns/reenqueuer.rb @@ -37,6 +37,7 @@ module Reenqueuer include ReenqueuerSleeper sidekiq_options retry: false + deduplicate :none end def perform(*args) @@ -52,7 +53,11 @@ module Reenqueuer private def reenqueue(*args) - self.class.perform_async(*args) if yield + result = yield + + self.class.perform_async(*args) if result + + result end # Override as needed diff --git a/app/workers/concerns/worker_context.rb b/app/workers/concerns/worker_context.rb index f2ff3ecfb6b..6acb9acceeb 100644 --- a/app/workers/concerns/worker_context.rb +++ b/app/workers/concerns/worker_context.rb @@ -5,7 +5,7 @@ module WorkerContext class_methods do def worker_context(attributes) - @worker_context = Gitlab::ApplicationContext.new(attributes) + @worker_context = Gitlab::ApplicationContext.new(**attributes) end def get_worker_context @@ -60,6 +60,6 @@ module WorkerContext end def with_context(context, &block) - Gitlab::ApplicationContext.new(context).use { yield(**context) } + Gitlab::ApplicationContext.new(**context).use { yield(**context) } end end diff --git a/app/workers/create_evidence_worker.rb b/app/workers/create_evidence_worker.rb deleted file mode 100644 index b18028e4114..00000000000 --- a/app/workers/create_evidence_worker.rb +++ /dev/null @@ -1,20 +0,0 @@ -# frozen_string_literal: true - -class CreateEvidenceWorker # rubocop:disable Scalability/IdempotentWorker - include ApplicationWorker - - feature_category :release_evidence - weight 2 - - # pipeline_id is optional for backward compatibility with existing jobs - # caller should always try to provide the pipeline and pass nil only - # if pipeline is absent - def perform(release_id, pipeline_id = nil) - release = Release.find_by_id(release_id) - return unless release - - pipeline = Ci::Pipeline.find_by_id(pipeline_id) - - ::Releases::CreateEvidenceService.new(release, pipeline: pipeline).execute - end -end diff --git a/app/workers/create_note_diff_file_worker.rb b/app/workers/create_note_diff_file_worker.rb index 8a1709f04e1..06790cc89d9 100644 --- a/app/workers/create_note_diff_file_worker.rb +++ b/app/workers/create_note_diff_file_worker.rb @@ -3,7 +3,7 @@ class CreateNoteDiffFileWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker - feature_category :source_code_management + feature_category :code_review def perform(diff_note_id) diff_note = DiffNote.find(diff_note_id) diff --git a/app/workers/delete_diff_files_worker.rb b/app/workers/delete_diff_files_worker.rb index a31cf650b83..289df8873ec 100644 --- a/app/workers/delete_diff_files_worker.rb +++ b/app/workers/delete_diff_files_worker.rb @@ -3,7 +3,7 @@ class DeleteDiffFilesWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker - feature_category :source_code_management + feature_category :code_review # rubocop: disable CodeReuse/ActiveRecord def perform(merge_request_diff_id) diff --git a/app/workers/environments/canary_ingress/update_worker.rb b/app/workers/environments/canary_ingress/update_worker.rb new file mode 100644 index 00000000000..53cc38e9eec --- /dev/null +++ b/app/workers/environments/canary_ingress/update_worker.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +module Environments + module CanaryIngress + class UpdateWorker + include ApplicationWorker + + sidekiq_options retry: false + idempotent! + worker_has_external_dependencies! + feature_category :continuous_delivery + + def perform(environment_id, params) + Environment.find_by_id(environment_id).try do |environment| + Environments::CanaryIngress::UpdateService + .new(environment.project, nil, params.with_indifferent_access) + .execute(environment) + end + end + end + end +end diff --git a/app/workers/gitlab/github_import/advance_stage_worker.rb b/app/workers/gitlab/github_import/advance_stage_worker.rb index 834c2f7791c..af406b32415 100644 --- a/app/workers/gitlab/github_import/advance_stage_worker.rb +++ b/app/workers/gitlab/github_import/advance_stage_worker.rb @@ -16,6 +16,8 @@ module Gitlab # The known importer stages and their corresponding Sidekiq workers. STAGES = { + pull_requests_merged_by: Stage::ImportPullRequestsMergedByWorker, + pull_request_reviews: Stage::ImportPullRequestsReviewsWorker, issues_and_diff_notes: Stage::ImportIssuesAndDiffNotesWorker, notes: Stage::ImportNotesWorker, lfs_objects: Stage::ImportLfsObjectsWorker, diff --git a/app/workers/gitlab/github_import/import_pull_request_merged_by_worker.rb b/app/workers/gitlab/github_import/import_pull_request_merged_by_worker.rb new file mode 100644 index 00000000000..79ef917bbc5 --- /dev/null +++ b/app/workers/gitlab/github_import/import_pull_request_merged_by_worker.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + class ImportPullRequestMergedByWorker # rubocop:disable Scalability/IdempotentWorker + include ObjectImporter + + def representation_class + Gitlab::GithubImport::Representation::PullRequest + end + + def importer_class + Importer::PullRequestMergedByImporter + end + + def counter_name + :github_importer_imported_pull_requests_merged_by + end + + def counter_description + 'The number of imported GitHub pull requests merged by' + end + end + end +end diff --git a/app/workers/gitlab/github_import/import_pull_request_review_worker.rb b/app/workers/gitlab/github_import/import_pull_request_review_worker.rb new file mode 100644 index 00000000000..b8516fb8670 --- /dev/null +++ b/app/workers/gitlab/github_import/import_pull_request_review_worker.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + class ImportPullRequestReviewWorker # rubocop:disable Scalability/IdempotentWorker + include ObjectImporter + + def representation_class + Gitlab::GithubImport::Representation::PullRequestReview + end + + def importer_class + Importer::PullRequestReviewImporter + end + + def counter_name + :github_importer_imported_pull_request_reviews + end + + def counter_description + 'The number of imported GitHub pull request reviews' + end + end + end +end diff --git a/app/workers/gitlab/github_import/import_pull_request_worker.rb b/app/workers/gitlab/github_import/import_pull_request_worker.rb index ec806ad170b..9560874f247 100644 --- a/app/workers/gitlab/github_import/import_pull_request_worker.rb +++ b/app/workers/gitlab/github_import/import_pull_request_worker.rb @@ -6,7 +6,7 @@ module Gitlab include ObjectImporter def representation_class - Representation::PullRequest + Gitlab::GithubImport::Representation::PullRequest end def importer_class diff --git a/app/workers/gitlab/github_import/stage/finish_import_worker.rb b/app/workers/gitlab/github_import/stage/finish_import_worker.rb index 73699a74a4a..058e1a0853d 100644 --- a/app/workers/gitlab/github_import/stage/finish_import_worker.rb +++ b/app/workers/gitlab/github_import/stage/finish_import_worker.rb @@ -20,12 +20,15 @@ module Gitlab def report_import_time(project) duration = Time.zone.now - project.created_at - path = project.full_path - histogram.observe({ project: path }, duration) + histogram.observe({ project: project.full_path }, duration) counter.increment - logger.info("GitHub importer finished for #{path} in #{duration.round(2)} seconds") + info( + project.id, + message: "GitHub project import finished", + duration_s: duration.round(2) + ) end def histogram diff --git a/app/workers/gitlab/github_import/stage/import_base_data_worker.rb b/app/workers/gitlab/github_import/stage/import_base_data_worker.rb index 11c2a2ac9b4..202bb335ca1 100644 --- a/app/workers/gitlab/github_import/stage/import_base_data_worker.rb +++ b/app/workers/gitlab/github_import/stage/import_base_data_worker.rb @@ -20,6 +20,7 @@ module Gitlab # project - An instance of Project. def import(client, project) IMPORTERS.each do |klass| + info(project.id, message: "starting importer", importer: klass.name) klass.new(project, client).execute end diff --git a/app/workers/gitlab/github_import/stage/import_issues_and_diff_notes_worker.rb b/app/workers/gitlab/github_import/stage/import_issues_and_diff_notes_worker.rb index 68b6e159fa4..486057804b4 100644 --- a/app/workers/gitlab/github_import/stage/import_issues_and_diff_notes_worker.rb +++ b/app/workers/gitlab/github_import/stage/import_issues_and_diff_notes_worker.rb @@ -19,6 +19,7 @@ module Gitlab # project - An instance of Project. def import(client, project) waiters = IMPORTERS.each_with_object({}) do |klass, hash| + info(project.id, message: "starting importer", importer: klass.name) waiter = klass.new(project, client).execute hash[waiter.key] = waiter.jobs_remaining end diff --git a/app/workers/gitlab/github_import/stage/import_lfs_objects_worker.rb b/app/workers/gitlab/github_import/stage/import_lfs_objects_worker.rb index a19df399969..de2a7f9fc29 100644 --- a/app/workers/gitlab/github_import/stage/import_lfs_objects_worker.rb +++ b/app/workers/gitlab/github_import/stage/import_lfs_objects_worker.rb @@ -16,6 +16,8 @@ module Gitlab # project - An instance of Project. def import(project) + info(project.id, message: "starting importer", importer: 'Importer::LfsObjectsImporter') + waiter = Importer::LfsObjectsImporter .new(project, nil) .execute diff --git a/app/workers/gitlab/github_import/stage/import_notes_worker.rb b/app/workers/gitlab/github_import/stage/import_notes_worker.rb index 49b9821cd45..e1da26a9d48 100644 --- a/app/workers/gitlab/github_import/stage/import_notes_worker.rb +++ b/app/workers/gitlab/github_import/stage/import_notes_worker.rb @@ -11,6 +11,7 @@ module Gitlab # client - An instance of Gitlab::GithubImport::Client. # project - An instance of Project. def import(client, project) + info(project.id, message: "starting importer", importer: 'Importer::NotesImporter') waiter = Importer::NotesImporter .new(project, client) .execute diff --git a/app/workers/gitlab/github_import/stage/import_pull_requests_merged_by_worker.rb b/app/workers/gitlab/github_import/stage/import_pull_requests_merged_by_worker.rb new file mode 100644 index 00000000000..3e15c346659 --- /dev/null +++ b/app/workers/gitlab/github_import/stage/import_pull_requests_merged_by_worker.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Stage + class ImportPullRequestsMergedByWorker # rubocop:disable Scalability/IdempotentWorker + include ApplicationWorker + include GithubImport::Queue + include StageMethods + + # client - An instance of Gitlab::GithubImport::Client. + # project - An instance of Project. + def import(client, project) + waiter = Importer::PullRequestsMergedByImporter + .new(project, client) + .execute + + project.import_state.refresh_jid_expiration + + AdvanceStageWorker.perform_async( + project.id, + { waiter.key => waiter.jobs_remaining }, + :pull_request_reviews + ) + end + end + end + end +end diff --git a/app/workers/gitlab/github_import/stage/import_pull_requests_reviews_worker.rb b/app/workers/gitlab/github_import/stage/import_pull_requests_reviews_worker.rb new file mode 100644 index 00000000000..0809d0b7c29 --- /dev/null +++ b/app/workers/gitlab/github_import/stage/import_pull_requests_reviews_worker.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Stage + class ImportPullRequestsReviewsWorker # rubocop:disable Scalability/IdempotentWorker + include ApplicationWorker + include GithubImport::Queue + include StageMethods + + # client - An instance of Gitlab::GithubImport::Client. + # project - An instance of Project. + def import(client, project) + waiter = + if Feature.enabled?(:github_import_pull_request_reviews, project, default_enabled: true) + waiter = Importer::PullRequestsReviewsImporter + .new(project, client) + .execute + + project.import_state.refresh_jid_expiration + + waiter + else + JobWaiter.new + end + + AdvanceStageWorker.perform_async( + project.id, + { waiter.key => waiter.jobs_remaining }, + :issues_and_diff_notes + ) + end + end + end + end +end diff --git a/app/workers/gitlab/github_import/stage/import_pull_requests_worker.rb b/app/workers/gitlab/github_import/stage/import_pull_requests_worker.rb index 3299db5653b..bf2defa6326 100644 --- a/app/workers/gitlab/github_import/stage/import_pull_requests_worker.rb +++ b/app/workers/gitlab/github_import/stage/import_pull_requests_worker.rb @@ -11,6 +11,7 @@ module Gitlab # client - An instance of Gitlab::GithubImport::Client. # project - An instance of Project. def import(client, project) + info(project.id, message: "starting importer", importer: 'Importer::PullRequestsImporter') waiter = Importer::PullRequestsImporter .new(project, client) .execute @@ -20,7 +21,7 @@ module Gitlab AdvanceStageWorker.perform_async( project.id, { waiter.key => waiter.jobs_remaining }, - :issues_and_diff_notes + :pull_requests_merged_by ) end end diff --git a/app/workers/gitlab/github_import/stage/import_repository_worker.rb b/app/workers/gitlab/github_import/stage/import_repository_worker.rb index cb9ef1cd198..3338f7e58c0 100644 --- a/app/workers/gitlab/github_import/stage/import_repository_worker.rb +++ b/app/workers/gitlab/github_import/stage/import_repository_worker.rb @@ -21,6 +21,7 @@ module Gitlab # expiration time. RefreshImportJidWorker.perform_in_the_future(project.id, jid) + info(project.id, message: "starting importer", importer: 'Importer::RepositoryImporter') importer = Importer::RepositoryImporter.new(project, client) return unless importer.execute diff --git a/app/workers/gitlab_performance_bar_stats_worker.rb b/app/workers/gitlab_performance_bar_stats_worker.rb new file mode 100644 index 00000000000..d63f8111864 --- /dev/null +++ b/app/workers/gitlab_performance_bar_stats_worker.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +class GitlabPerformanceBarStatsWorker + include ApplicationWorker + + LEASE_KEY = 'gitlab:performance_bar_stats' + LEASE_TIMEOUT = 600 + WORKER_DELAY = 120 + STATS_KEY = 'performance_bar_stats:pending_request_ids' + + feature_category :metrics + idempotent! + + def perform(lease_uuid) + Gitlab::Redis::SharedState.with do |redis| + request_ids = fetch_request_ids(redis, lease_uuid) + stats = Gitlab::PerformanceBar::Stats.new(redis) + + request_ids.each do |id| + stats.process(id) + end + end + end + + private + + def fetch_request_ids(redis, lease_uuid) + ids = redis.smembers(STATS_KEY) + redis.del(STATS_KEY) + Gitlab::ExclusiveLease.cancel(LEASE_KEY, lease_uuid) + + ids + end +end diff --git a/app/workers/gitlab_usage_ping_worker.rb b/app/workers/gitlab_usage_ping_worker.rb index a696c6e746a..1bb600bbd13 100644 --- a/app/workers/gitlab_usage_ping_worker.rb +++ b/app/workers/gitlab_usage_ping_worker.rb @@ -13,6 +13,10 @@ class GitlabUsagePingWorker # rubocop:disable Scalability/IdempotentWorker sidekiq_retry_in { |count| (count + 1) * 8.hours.to_i } def perform + # Disable usage ping for GitLab.com + # See https://gitlab.com/gitlab-org/gitlab/-/issues/292929 for details + return if Gitlab.com? + # Multiple Sidekiq workers could run this. We should only do this at most once a day. in_lock(LEASE_KEY, ttl: LEASE_TIMEOUT) do # Splay the request over a minute to avoid thundering herd problems. diff --git a/app/workers/import_issues_csv_worker.rb b/app/workers/import_issues_csv_worker.rb index c7b5f8cd0a7..521e5b8fbc2 100644 --- a/app/workers/import_issues_csv_worker.rb +++ b/app/workers/import_issues_csv_worker.rb @@ -3,6 +3,7 @@ class ImportIssuesCsvWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker + idempotent! feature_category :issue_tracking worker_resource_boundary :cpu weight 2 @@ -12,13 +13,15 @@ class ImportIssuesCsvWorker # rubocop:disable Scalability/IdempotentWorker end def perform(current_user_id, project_id, upload_id) - @user = User.find(current_user_id) - @project = Project.find(project_id) - @upload = Upload.find(upload_id) + user = User.find(current_user_id) + project = Project.find(project_id) + upload = Upload.find(upload_id) - importer = Issues::ImportCsvService.new(@user, @project, @upload.retrieve_uploader) + importer = Issues::ImportCsvService.new(user, project, upload.retrieve_uploader) importer.execute - @upload.destroy + upload.destroy + rescue ActiveRecord::RecordNotFound + # Resources have been removed, job should not be retried end end diff --git a/app/workers/jira_connect/sync_branch_worker.rb b/app/workers/jira_connect/sync_branch_worker.rb index 4c1c987353d..d7e773b0861 100644 --- a/app/workers/jira_connect/sync_branch_worker.rb +++ b/app/workers/jira_connect/sync_branch_worker.rb @@ -7,6 +7,7 @@ module JiraConnect queue_namespace :jira_connect feature_category :integrations loggable_arguments 1, 2 + worker_has_external_dependencies! def perform(project_id, branch_name, commit_shas, update_sequence_id = nil) project = Project.find_by_id(project_id) diff --git a/app/workers/jira_connect/sync_builds_worker.rb b/app/workers/jira_connect/sync_builds_worker.rb new file mode 100644 index 00000000000..c1c749f6041 --- /dev/null +++ b/app/workers/jira_connect/sync_builds_worker.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +module JiraConnect + class SyncBuildsWorker + include ApplicationWorker + + idempotent! + worker_has_external_dependencies! + + queue_namespace :jira_connect + feature_category :integrations + + def perform(pipeline_id, sequence_id) + pipeline = Ci::Pipeline.find_by_id(pipeline_id) + + return unless pipeline + return unless Feature.enabled?(:jira_sync_builds, pipeline.project) + + ::JiraConnect::SyncService + .new(pipeline.project) + .execute(pipelines: [pipeline], update_sequence_id: sequence_id) + end + end +end diff --git a/app/workers/jira_connect/sync_merge_request_worker.rb b/app/workers/jira_connect/sync_merge_request_worker.rb index f45ab38f35d..6ef426790b3 100644 --- a/app/workers/jira_connect/sync_merge_request_worker.rb +++ b/app/workers/jira_connect/sync_merge_request_worker.rb @@ -7,6 +7,8 @@ module JiraConnect queue_namespace :jira_connect feature_category :integrations + worker_has_external_dependencies! + def perform(merge_request_id, update_sequence_id = nil) merge_request = MergeRequest.find_by_id(merge_request_id) diff --git a/app/workers/member_invitation_reminder_emails_worker.rb b/app/workers/member_invitation_reminder_emails_worker.rb index 50f583005c0..971d6abaa51 100644 --- a/app/workers/member_invitation_reminder_emails_worker.rb +++ b/app/workers/member_invitation_reminder_emails_worker.rb @@ -8,8 +8,6 @@ class MemberInvitationReminderEmailsWorker # rubocop:disable Scalability/Idempot urgency :low def perform - return unless Gitlab::Experimentation.enabled?(:invitation_reminders) - Member.not_accepted_invitations.not_expired.last_ten_days_excluding_today.find_in_batches do |invitations| invitations.each do |invitation| Members::InvitationReminderEmailService.new(invitation).execute diff --git a/app/workers/merge_request_cleanup_refs_worker.rb b/app/workers/merge_request_cleanup_refs_worker.rb index 37774658ba8..6b991a2253f 100644 --- a/app/workers/merge_request_cleanup_refs_worker.rb +++ b/app/workers/merge_request_cleanup_refs_worker.rb @@ -3,7 +3,7 @@ class MergeRequestCleanupRefsWorker include ApplicationWorker - feature_category :source_code_management + feature_category :code_review idempotent! def perform(merge_request_id) diff --git a/app/workers/merge_request_mergeability_check_worker.rb b/app/workers/merge_request_mergeability_check_worker.rb index 1a84efb4e52..70d5f49d70e 100644 --- a/app/workers/merge_request_mergeability_check_worker.rb +++ b/app/workers/merge_request_mergeability_check_worker.rb @@ -3,7 +3,7 @@ class MergeRequestMergeabilityCheckWorker include ApplicationWorker - feature_category :source_code_management + feature_category :code_review idempotent! def perform(merge_request_id) diff --git a/app/workers/migrate_external_diffs_worker.rb b/app/workers/migrate_external_diffs_worker.rb index 0a95f40aa8f..3ef399bd9fc 100644 --- a/app/workers/migrate_external_diffs_worker.rb +++ b/app/workers/migrate_external_diffs_worker.rb @@ -3,7 +3,7 @@ class MigrateExternalDiffsWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker - feature_category :source_code_management + feature_category :code_review def perform(merge_request_diff_id) diff = MergeRequestDiff.find_by_id(merge_request_diff_id) diff --git a/app/workers/namespaces/onboarding_user_added_worker.rb b/app/workers/namespaces/onboarding_user_added_worker.rb new file mode 100644 index 00000000000..02608268d6f --- /dev/null +++ b/app/workers/namespaces/onboarding_user_added_worker.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module Namespaces + class OnboardingUserAddedWorker + include ApplicationWorker + + feature_category :users + urgency :low + + idempotent! + + def perform(namespace_id) + namespace = Namespace.find(namespace_id) + OnboardingProgressService.new(namespace).execute(action: :user_added) + end + end +end diff --git a/app/workers/new_merge_request_worker.rb b/app/workers/new_merge_request_worker.rb index f672d37a83e..2d28561488b 100644 --- a/app/workers/new_merge_request_worker.rb +++ b/app/workers/new_merge_request_worker.rb @@ -4,7 +4,7 @@ class NewMergeRequestWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker include NewIssuable - feature_category :source_code_management + feature_category :code_review urgency :high worker_resource_boundary :cpu weight 2 diff --git a/app/workers/project_cache_worker.rb b/app/workers/project_cache_worker.rb index b114c67de47..8a9c166e5df 100644 --- a/app/workers/project_cache_worker.rb +++ b/app/workers/project_cache_worker.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true # Worker for updating any project specific caches. -class ProjectCacheWorker # rubocop:disable Scalability/IdempotentWorker +class ProjectCacheWorker include ApplicationWorker LEASE_TIMEOUT = 15.minutes.to_i @@ -9,6 +9,7 @@ class ProjectCacheWorker # rubocop:disable Scalability/IdempotentWorker feature_category :source_code_management urgency :high loggable_arguments 1, 2, 3 + idempotent! # project_id - The ID of the project for which to flush the cache. # files - An Array containing extra types of files to refresh such as diff --git a/app/workers/project_schedule_bulk_repository_shard_moves_worker.rb b/app/workers/project_schedule_bulk_repository_shard_moves_worker.rb new file mode 100644 index 00000000000..4d2a6b47e3c --- /dev/null +++ b/app/workers/project_schedule_bulk_repository_shard_moves_worker.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +class ProjectScheduleBulkRepositoryShardMovesWorker + include ApplicationWorker + + idempotent! + feature_category :gitaly + urgency :throttled + + def perform(source_storage_name, destination_storage_name = nil) + Projects::ScheduleBulkRepositoryShardMovesService.new.execute(source_storage_name, destination_storage_name) + end +end diff --git a/app/workers/purge_dependency_proxy_cache_worker.rb b/app/workers/purge_dependency_proxy_cache_worker.rb index 594cdd3ed11..b4c88592543 100644 --- a/app/workers/purge_dependency_proxy_cache_worker.rb +++ b/app/workers/purge_dependency_proxy_cache_worker.rb @@ -15,6 +15,7 @@ class PurgeDependencyProxyCacheWorker return unless valid? @group.dependency_proxy_blobs.destroy_all # rubocop:disable Cop/DestroyAll + @group.dependency_proxy_manifests.destroy_all # rubocop:disable Cop/DestroyAll end private diff --git a/app/workers/releases/create_evidence_worker.rb b/app/workers/releases/create_evidence_worker.rb new file mode 100644 index 00000000000..db75fae1639 --- /dev/null +++ b/app/workers/releases/create_evidence_worker.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module Releases + class CreateEvidenceWorker # rubocop:disable Scalability/IdempotentWorker + include ApplicationWorker + include CronjobQueue # rubocop:disable Scalability/CronWorkerContext + + feature_category :release_evidence + + # pipeline_id is optional for backward compatibility with existing jobs + # caller should always try to provide the pipeline and pass nil only + # if pipeline is absent + def perform(release_id, pipeline_id = nil) + release = Release.find_by_id(release_id) + + return unless release + + pipeline = Ci::Pipeline.find_by_id(pipeline_id) + + ::Releases::CreateEvidenceService.new(release, pipeline: pipeline).execute + end + end +end diff --git a/app/workers/releases/manage_evidence_worker.rb b/app/workers/releases/manage_evidence_worker.rb new file mode 100644 index 00000000000..8a925d22cea --- /dev/null +++ b/app/workers/releases/manage_evidence_worker.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +module Releases + class ManageEvidenceWorker # rubocop:disable Scalability/IdempotentWorker + include ApplicationWorker + include CronjobQueue # rubocop:disable Scalability/CronWorkerContext + + feature_category :release_evidence + + def perform + releases = Release.without_evidence.released_within_2hrs + + releases.each do |release| + project = release.project + params = { tag: release.tag } + + evidence_pipeline = Releases::EvidencePipelineFinder.new(project, params).execute + + # perform_at released_at + ::Releases::CreateEvidenceWorker.perform_async(release.id, evidence_pipeline&.id) + end + end + end +end diff --git a/app/workers/repository_import_worker.rb b/app/workers/repository_import_worker.rb index 51fe60e25fc..90764d7374d 100644 --- a/app/workers/repository_import_worker.rb +++ b/app/workers/repository_import_worker.rb @@ -30,7 +30,7 @@ class RepositoryImportWorker # rubocop:disable Scalability/IdempotentWorker return if service.async? if result[:status] == :error - fail_import(result[:message]) if template_import? + fail_import(result[:message]) raise result[:message] end diff --git a/app/workers/repository_update_remote_mirror_worker.rb b/app/workers/repository_update_remote_mirror_worker.rb index 21b5916f459..483aae84a3b 100644 --- a/app/workers/repository_update_remote_mirror_worker.rb +++ b/app/workers/repository_update_remote_mirror_worker.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -class RepositoryUpdateRemoteMirrorWorker # rubocop:disable Scalability/IdempotentWorker +class RepositoryUpdateRemoteMirrorWorker UpdateError = Class.new(StandardError) include ApplicationWorker @@ -11,6 +11,7 @@ class RepositoryUpdateRemoteMirrorWorker # rubocop:disable Scalability/Idempoten sidekiq_options retry: 3, dead: false feature_category :source_code_management loggable_arguments 1 + idempotent! LOCK_WAIT_TIME = 30.seconds MAX_TRIES = 3 diff --git a/app/workers/schedule_merge_request_cleanup_refs_worker.rb b/app/workers/schedule_merge_request_cleanup_refs_worker.rb index 17cabba4278..59b8993f78f 100644 --- a/app/workers/schedule_merge_request_cleanup_refs_worker.rb +++ b/app/workers/schedule_merge_request_cleanup_refs_worker.rb @@ -4,7 +4,7 @@ class ScheduleMergeRequestCleanupRefsWorker include ApplicationWorker include CronjobQueue # rubocop:disable Scalability/CronWorkerContext - feature_category :source_code_management + feature_category :code_review idempotent! # Based on existing data, MergeRequestCleanupRefsWorker can run 3 jobs per diff --git a/app/workers/schedule_migrate_external_diffs_worker.rb b/app/workers/schedule_migrate_external_diffs_worker.rb index 4e7b60c4ab7..70e4d56562b 100644 --- a/app/workers/schedule_migrate_external_diffs_worker.rb +++ b/app/workers/schedule_migrate_external_diffs_worker.rb @@ -10,7 +10,7 @@ class ScheduleMigrateExternalDiffsWorker # rubocop:disable Scalability/Idempoten include Gitlab::ExclusiveLeaseHelpers - feature_category :source_code_management + feature_category :code_review def perform in_lock(self.class.name.underscore, ttl: 2.hours, retries: 0) do diff --git a/app/workers/stuck_merge_jobs_worker.rb b/app/workers/stuck_merge_jobs_worker.rb index 0f9b4ddb980..bea9d67b3e8 100644 --- a/app/workers/stuck_merge_jobs_worker.rb +++ b/app/workers/stuck_merge_jobs_worker.rb @@ -4,7 +4,7 @@ class StuckMergeJobsWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker include CronjobQueue # rubocop:disable Scalability/CronWorkerContext - feature_category :source_code_management + feature_category :code_review def self.logger Gitlab::AppLogger diff --git a/app/workers/trending_projects_worker.rb b/app/workers/trending_projects_worker.rb index eb1a7f4fef9..5876cfb1fe7 100644 --- a/app/workers/trending_projects_worker.rb +++ b/app/workers/trending_projects_worker.rb @@ -2,10 +2,6 @@ class TrendingProjectsWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker - # rubocop:disable Scalability/CronWorkerContext - # This worker does not perform work scoped to a context - include CronjobQueue - # rubocop:enable Scalability/CronWorkerContext include CronjobQueue # rubocop:disable Scalability/CronWorkerContext feature_category :source_code_management diff --git a/app/workers/update_merge_requests_worker.rb b/app/workers/update_merge_requests_worker.rb index 402c1777662..46cb32e7f08 100644 --- a/app/workers/update_merge_requests_worker.rb +++ b/app/workers/update_merge_requests_worker.rb @@ -3,7 +3,7 @@ class UpdateMergeRequestsWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker - feature_category :source_code_management + feature_category :code_review urgency :high worker_resource_boundary :cpu weight 3 |