diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2022-08-18 11:17:02 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2022-08-18 11:17:02 +0300 |
commit | b39512ed755239198a9c294b6a45e65c05900235 (patch) | |
tree | d234a3efade1de67c46b9e5a38ce813627726aa7 /app/workers | |
parent | d31474cf3b17ece37939d20082b07f6657cc79a9 (diff) |
Add latest changes from gitlab-org/gitlab@15-3-stable-eev15.3.0-rc42
Diffstat (limited to 'app/workers')
26 files changed, 404 insertions, 125 deletions
diff --git a/app/workers/all_queues.yml b/app/workers/all_queues.yml index 966a1202db2..8bba5e36b52 100644 --- a/app/workers/all_queues.yml +++ b/app/workers/all_queues.yml @@ -1551,15 +1551,6 @@ :weight: 1 :idempotent: false :tags: [] -- :name: pipeline_background:archive_trace - :worker_name: ArchiveTraceWorker - :feature_category: :continuous_integration - :has_external_dependencies: false - :urgency: :low - :resource_boundary: :unknown - :weight: 1 - :idempotent: false - :tags: [] - :name: pipeline_background:ci_archive_trace :worker_name: Ci::ArchiveTraceWorker :feature_category: :continuous_integration @@ -1650,6 +1641,15 @@ :weight: 1 :idempotent: true :tags: [] +- :name: pipeline_background:ci_track_failed_build + :worker_name: Ci::TrackFailedBuildWorker + :feature_category: :static_application_security_testing + :has_external_dependencies: true + :urgency: :low + :resource_boundary: :cpu + :weight: 1 + :idempotent: true + :tags: [] - :name: pipeline_creation:ci_external_pull_requests_create_pipeline :worker_name: Ci::ExternalPullRequests::CreatePipelineWorker :feature_category: :continuous_integration @@ -1776,15 +1776,6 @@ :weight: 2 :idempotent: false :tags: [] -- :name: pipeline_processing:build_finished - :worker_name: BuildFinishedWorker - :feature_category: :continuous_integration - :has_external_dependencies: false - :urgency: :high - :resource_boundary: :cpu - :weight: 5 - :idempotent: false - :tags: [] - :name: pipeline_processing:build_queue :worker_name: BuildQueueWorker :feature_category: :continuous_integration @@ -2109,6 +2100,15 @@ :weight: 2 :idempotent: false :tags: [] +- :name: ci_cancel_pipeline + :worker_name: Ci::CancelPipelineWorker + :feature_category: :continuous_integration + :has_external_dependencies: false + :urgency: :high + :resource_boundary: :unknown + :weight: 1 + :idempotent: true + :tags: [] - :name: ci_delete_objects :worker_name: Ci::DeleteObjectsWorker :feature_category: :continuous_integration @@ -2127,6 +2127,15 @@ :weight: 1 :idempotent: true :tags: [] +- :name: ci_runners_process_runner_version_update + :worker_name: Ci::Runners::ProcessRunnerVersionUpdateWorker + :feature_category: :runner_fleet + :has_external_dependencies: false + :urgency: :low + :resource_boundary: :unknown + :weight: 1 + :idempotent: true + :tags: [] - :name: create_commit_signature :worker_name: CreateCommitSignatureWorker :feature_category: :source_code_management @@ -2252,8 +2261,7 @@ :resource_boundary: :unknown :weight: 2 :idempotent: false - :tags: - - :needs_own_queue + :tags: [] - :name: emails_on_push :worker_name: EmailsOnPushWorker :feature_category: :source_code_management @@ -2551,6 +2559,24 @@ :weight: 1 :idempotent: true :tags: [] +- :name: merge_requests_create_approval_event + :worker_name: MergeRequests::CreateApprovalEventWorker + :feature_category: :code_review + :has_external_dependencies: false + :urgency: :low + :resource_boundary: :unknown + :weight: 1 + :idempotent: true + :tags: [] +- :name: merge_requests_create_approval_note + :worker_name: MergeRequests::CreateApprovalNoteWorker + :feature_category: :code_review + :has_external_dependencies: false + :urgency: :low + :resource_boundary: :unknown + :weight: 1 + :idempotent: true + :tags: [] - :name: merge_requests_delete_source_branch :worker_name: MergeRequests::DeleteSourceBranchWorker :feature_category: :source_code_management @@ -2560,6 +2586,15 @@ :weight: 1 :idempotent: true :tags: [] +- :name: merge_requests_execute_approval_hooks + :worker_name: MergeRequests::ExecuteApprovalHooksWorker + :feature_category: :code_review + :has_external_dependencies: true + :urgency: :low + :resource_boundary: :unknown + :weight: 1 + :idempotent: true + :tags: [] - :name: merge_requests_handle_assignees_change :worker_name: MergeRequests::HandleAssigneesChangeWorker :feature_category: :code_review @@ -2578,6 +2613,15 @@ :weight: 1 :idempotent: true :tags: [] +- :name: merge_requests_resolve_todos_after_approval + :worker_name: MergeRequests::ResolveTodosAfterApprovalWorker + :feature_category: :code_review + :has_external_dependencies: false + :urgency: :low + :resource_boundary: :unknown + :weight: 1 + :idempotent: true + :tags: [] - :name: merge_requests_update_head_pipeline :worker_name: MergeRequests::UpdateHeadPipelineWorker :feature_category: :code_review @@ -2812,6 +2856,15 @@ :weight: 1 :idempotent: false :tags: [] +- :name: projects_import_export_relation_export + :worker_name: Projects::ImportExport::RelationExportWorker + :feature_category: :importers + :has_external_dependencies: false + :urgency: :low + :resource_boundary: :memory + :weight: 1 + :idempotent: true + :tags: [] - :name: projects_inactive_projects_deletion_notification :worker_name: Projects::InactiveProjectsDeletionNotificationWorker :feature_category: :compliance_management @@ -3018,8 +3071,7 @@ :resource_boundary: :unknown :weight: 2 :idempotent: false - :tags: - - :needs_own_queue + :tags: [] - :name: snippets_schedule_bulk_repository_shard_moves :worker_name: Snippets::ScheduleBulkRepositoryShardMovesWorker :feature_category: :gitaly diff --git a/app/workers/archive_trace_worker.rb b/app/workers/archive_trace_worker.rb deleted file mode 100644 index ecde05f94dc..00000000000 --- a/app/workers/archive_trace_worker.rb +++ /dev/null @@ -1,5 +0,0 @@ -# frozen_string_literal: true - -class ArchiveTraceWorker < ::Ci::ArchiveTraceWorker # rubocop:disable Scalability/IdempotentWorker - # DEPRECATED: Not triggered since https://gitlab.com/gitlab-org/gitlab/-/merge_requests/64934/ -end diff --git a/app/workers/build_finished_worker.rb b/app/workers/build_finished_worker.rb deleted file mode 100644 index 0d41f7b9438..00000000000 --- a/app/workers/build_finished_worker.rb +++ /dev/null @@ -1,9 +0,0 @@ -# frozen_string_literal: true - -class BuildFinishedWorker < ::Ci::BuildFinishedWorker # rubocop:disable Scalability/IdempotentWorker - # DEPRECATED: Not triggered since https://gitlab.com/gitlab-org/gitlab/-/merge_requests/64934/ - - # We need to explicitly specify these settings. They aren't inheriting from the parent class. - urgency :high - worker_resource_boundary :cpu -end diff --git a/app/workers/build_hooks_worker.rb b/app/workers/build_hooks_worker.rb index 5c08344bfe3..2c62aed72d6 100644 --- a/app/workers/build_hooks_worker.rb +++ b/app/workers/build_hooks_worker.rb @@ -13,9 +13,9 @@ class BuildHooksWorker # rubocop:disable Scalability/IdempotentWorker # rubocop: disable CodeReuse/ActiveRecord def perform(build_id) - Ci::Build.includes({ runner: :tags }) - .find_by_id(build_id) - .try(:execute_hooks) + build = Ci::Build.find_by_id(build_id) + + build.execute_hooks if build end # rubocop: enable CodeReuse/ActiveRecord diff --git a/app/workers/ci/build_finished_worker.rb b/app/workers/ci/build_finished_worker.rb index 25c7637a79f..36a50735fed 100644 --- a/app/workers/ci/build_finished_worker.rb +++ b/app/workers/ci/build_finished_worker.rb @@ -36,8 +36,7 @@ module Ci build.update_coverage Ci::BuildReportResultService.new.execute(build) - # We execute these async as these are independent operations. - BuildHooksWorker.perform_async(build) + build.feature_flagged_execute_hooks ChatNotificationWorker.perform_async(build.id) if build.pipeline.chat? build.track_deployment_usage build.track_verify_usage diff --git a/app/workers/ci/cancel_pipeline_worker.rb b/app/workers/ci/cancel_pipeline_worker.rb new file mode 100644 index 00000000000..147839a0625 --- /dev/null +++ b/app/workers/ci/cancel_pipeline_worker.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module Ci + class CancelPipelineWorker + include ApplicationWorker + + # lots of updates to ci_builds + data_consistency :always + feature_category :continuous_integration + idempotent! + deduplicate :until_executed + urgency :high + + def perform(pipeline_id, auto_canceled_by_pipeline_id) + ::Ci::Pipeline.find_by_id(pipeline_id).try do |pipeline| + pipeline.cancel_running( + # cascade_to_children is false because we iterate through children + # we also cancel bridges prior to prevent more children + cascade_to_children: false, + auto_canceled_by_pipeline_id: auto_canceled_by_pipeline_id + ) + end + end + end +end diff --git a/app/workers/ci/runners/process_runner_version_update_worker.rb b/app/workers/ci/runners/process_runner_version_update_worker.rb new file mode 100644 index 00000000000..f1ad0c8563e --- /dev/null +++ b/app/workers/ci/runners/process_runner_version_update_worker.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module Ci + module Runners + class ProcessRunnerVersionUpdateWorker + include ApplicationWorker + + data_consistency :always + + feature_category :runner_fleet + urgency :low + + idempotent! + deduplicate :until_executing + + def perform(version) + result = ::Ci::Runners::ProcessRunnerVersionUpdateService.new(version).execute + + result.to_h.slice(:status, :message, :upgrade_status).each do |key, value| + log_extra_metadata_on_done(key, value) + end + end + end + end +end diff --git a/app/workers/ci/runners/reconcile_existing_runner_versions_cron_worker.rb b/app/workers/ci/runners/reconcile_existing_runner_versions_cron_worker.rb index 035b2563e56..69ab477c80a 100644 --- a/app/workers/ci/runners/reconcile_existing_runner_versions_cron_worker.rb +++ b/app/workers/ci/runners/reconcile_existing_runner_versions_cron_worker.rb @@ -12,11 +12,25 @@ module Ci feature_category :runner_fleet urgency :low + deduplicate :until_executed idempotent! - def perform + def perform(cronjob_scheduled = true) + if cronjob_scheduled + # Introduce some randomness across the day so that instances don't all hit the GitLab Releases API + # around the same time of day + period = rand(0..12.hours.in_seconds) + self.class.perform_in(period, false) + + Sidekiq.logger.info( + class: self.class.name, + message: "rescheduled job for #{period.seconds.from_now}") + + return + end + result = ::Ci::Runners::ReconcileExistingRunnerVersionsService.new.execute - result.each { |key, value| log_extra_metadata_on_done(key, value) } + result.payload.each { |key, value| log_extra_metadata_on_done(key, value) } end end end diff --git a/app/workers/ci/track_failed_build_worker.rb b/app/workers/ci/track_failed_build_worker.rb new file mode 100644 index 00000000000..2ad948876ac --- /dev/null +++ b/app/workers/ci/track_failed_build_worker.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +# Worker for tracking exit codes of failed CI jobs +module Ci + class TrackFailedBuildWorker # rubocop:disable Scalability/IdempotentWorker + include ApplicationWorker + include PipelineBackgroundQueue + + feature_category :static_application_security_testing + + urgency :low + data_consistency :sticky + worker_resource_boundary :cpu + idempotent! + worker_has_external_dependencies! + + def perform(build_id, exit_code, failure_reason) + ::Ci::Build.find_by_id(build_id).try do |build| + ::Ci::TrackFailedBuildService.new( + build: build, + exit_code: exit_code, + failure_reason: failure_reason).execute + end + end + end +end diff --git a/app/workers/concerns/waitable_worker.rb b/app/workers/concerns/waitable_worker.rb index 336d60d46ac..9300c2a5790 100644 --- a/app/workers/concerns/waitable_worker.rb +++ b/app/workers/concerns/waitable_worker.rb @@ -7,7 +7,7 @@ module WaitableWorker # Schedules multiple jobs and waits for them to be completed. def bulk_perform_and_wait(args_list) # Short-circuit: it's more efficient to do small numbers of jobs inline - if args_list.size == 1 + if args_list.size == 1 && !always_async_project_authorizations_refresh? return bulk_perform_inline(args_list) end @@ -29,6 +29,10 @@ module WaitableWorker bulk_perform_async(failed) if failed.present? end + + def always_async_project_authorizations_refresh? + Feature.enabled?(:always_async_project_authorizations_refresh) + end end def perform(*args) diff --git a/app/workers/email_receiver_worker.rb b/app/workers/email_receiver_worker.rb index 54689df4d7b..339383476be 100644 --- a/app/workers/email_receiver_worker.rb +++ b/app/workers/email_receiver_worker.rb @@ -11,9 +11,6 @@ class EmailReceiverWorker # rubocop:disable Scalability/IdempotentWorker urgency :high weight 2 - # https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/1263 - tags :needs_own_queue - attr_accessor :raw def perform(raw) diff --git a/app/workers/emails_on_push_worker.rb b/app/workers/emails_on_push_worker.rb index d7bd8207f06..5cc9bb6954e 100644 --- a/app/workers/emails_on_push_worker.rb +++ b/app/workers/emails_on_push_worker.rb @@ -17,8 +17,8 @@ class EmailsOnPushWorker # rubocop:disable Scalability/IdempotentWorker def perform(project_id, recipients, push_data, options = {}) options.symbolize_keys! options.reverse_merge!( - send_from_committer_email: false, - disable_diffs: false + send_from_committer_email: false, + disable_diffs: false ) send_from_committer_email = options[:send_from_committer_email] disable_diffs = options[:disable_diffs] @@ -64,14 +64,14 @@ class EmailsOnPushWorker # rubocop:disable Scalability/IdempotentWorker send_email( recipient, project_id, - author_id: author_id, - ref: ref, - action: action, - compare: compare, - reverse_compare: reverse_compare, - diff_refs: diff_refs, + author_id: author_id, + ref: ref, + action: action, + compare: compare, + reverse_compare: reverse_compare, + diff_refs: diff_refs, send_from_committer_email: send_from_committer_email, - disable_diffs: disable_diffs + disable_diffs: disable_diffs ) # These are input errors and won't be corrected even if Sidekiq retries diff --git a/app/workers/gitlab/github_import/stage/import_issue_events_worker.rb b/app/workers/gitlab/github_import/stage/import_issue_events_worker.rb index 8155b910677..0ec0a1b58d2 100644 --- a/app/workers/gitlab/github_import/stage/import_issue_events_worker.rb +++ b/app/workers/gitlab/github_import/stage/import_issue_events_worker.rb @@ -15,32 +15,34 @@ module Gitlab # client - An instance of Gitlab::GithubImport::Client. # project - An instance of Project. def import(client, project) - importer = ::Gitlab::GithubImport::Importer::SingleEndpointIssueEventsImporter - return skip_to_next_stage(project, importer) if feature_disabled?(project) + importer = importer_class(project) + return skip_to_next_stage(project) if importer.nil? - start_importer(project, importer, client) + info(project.id, message: "starting importer", importer: importer.name) + waiter = importer.new(project, client).execute + move_to_next_stage(project, { waiter.key => waiter.jobs_remaining }) end private - def start_importer(project, importer, client) - info(project.id, message: "starting importer", importer: importer.name) - waiter = importer.new(project, client).execute - move_to_next_stage(project, waiter.key => waiter.jobs_remaining) + def importer_class(project) + if Feature.enabled?(:github_importer_single_endpoint_issue_events_import, project.group, type: :ops) + ::Gitlab::GithubImport::Importer::SingleEndpointIssueEventsImporter + elsif Feature.enabled?(:github_importer_issue_events_import, project.group, type: :ops) + ::Gitlab::GithubImport::Importer::IssueEventsImporter + else + nil + end end - def skip_to_next_stage(project, importer) - info(project.id, message: "skipping importer", importer: importer.name) + def skip_to_next_stage(project) + info(project.id, message: "skipping importer", importer: "IssueEventsImporter") move_to_next_stage(project) end def move_to_next_stage(project, waiters = {}) AdvanceStageWorker.perform_async(project.id, waiters, :notes) end - - def feature_disabled?(project) - Feature.disabled?(:github_importer_issue_events_import, project.group, type: :ops) - end end end end diff --git a/app/workers/merge_requests/create_approval_event_worker.rb b/app/workers/merge_requests/create_approval_event_worker.rb new file mode 100644 index 00000000000..9b1a3c262e4 --- /dev/null +++ b/app/workers/merge_requests/create_approval_event_worker.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +module MergeRequests + class CreateApprovalEventWorker + include Gitlab::EventStore::Subscriber + + data_consistency :always + feature_category :code_review + urgency :low + idempotent! + + def handle_event(event) + current_user_id = event.data[:current_user_id] + merge_request_id = event.data[:merge_request_id] + current_user = User.find_by_id(current_user_id) + + unless current_user + logger.info(structured_payload(message: 'Current user not found.', current_user_id: current_user_id)) + return + end + + merge_request = MergeRequest.find_by_id(merge_request_id) + + unless merge_request + logger.info(structured_payload(message: 'Merge request not found.', merge_request_id: merge_request_id)) + return + end + + ::MergeRequests::CreateApprovalEventService + .new(project: merge_request.project, current_user: current_user) + .execute(merge_request) + end + end +end diff --git a/app/workers/merge_requests/create_approval_note_worker.rb b/app/workers/merge_requests/create_approval_note_worker.rb new file mode 100644 index 00000000000..841431f6a9d --- /dev/null +++ b/app/workers/merge_requests/create_approval_note_worker.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +module MergeRequests + class CreateApprovalNoteWorker + include Gitlab::EventStore::Subscriber + + data_consistency :always + feature_category :code_review + urgency :low + idempotent! + + def handle_event(event) + current_user_id = event.data[:current_user_id] + merge_request_id = event.data[:merge_request_id] + current_user = User.find_by_id(current_user_id) + + unless current_user + logger.info(structured_payload(message: 'Current user not found.', current_user_id: current_user_id)) + return + end + + merge_request = MergeRequest.find_by_id(merge_request_id) + + unless merge_request + logger.info(structured_payload(message: 'Merge request not found.', merge_request_id: merge_request_id)) + return + end + + SystemNoteService.approve_mr(merge_request, current_user) + end + end +end diff --git a/app/workers/merge_requests/execute_approval_hooks_worker.rb b/app/workers/merge_requests/execute_approval_hooks_worker.rb new file mode 100644 index 00000000000..81eca425a38 --- /dev/null +++ b/app/workers/merge_requests/execute_approval_hooks_worker.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +module MergeRequests + class ExecuteApprovalHooksWorker + include Gitlab::EventStore::Subscriber + + data_consistency :always + feature_category :code_review + urgency :low + idempotent! + + # MergeRequests::ExecuteApprovalHooksService execute webhooks which are treated as external dependencies + worker_has_external_dependencies! + + def handle_event(event) + current_user_id = event.data[:current_user_id] + merge_request_id = event.data[:merge_request_id] + current_user = User.find_by_id(current_user_id) + + unless current_user + logger.info(structured_payload(message: 'Current user not found.', current_user_id: current_user_id)) + return + end + + merge_request = MergeRequest.find_by_id(merge_request_id) + + unless merge_request + logger.info(structured_payload(message: 'Merge request not found.', merge_request_id: merge_request_id)) + return + end + + ::MergeRequests::ExecuteApprovalHooksService + .new(project: merge_request.project, current_user: current_user) + .execute(merge_request) + end + end +end diff --git a/app/workers/merge_requests/resolve_todos_after_approval_worker.rb b/app/workers/merge_requests/resolve_todos_after_approval_worker.rb new file mode 100644 index 00000000000..7d9c76ea872 --- /dev/null +++ b/app/workers/merge_requests/resolve_todos_after_approval_worker.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +module MergeRequests + class ResolveTodosAfterApprovalWorker + include Gitlab::EventStore::Subscriber + + data_consistency :always + feature_category :code_review + urgency :low + idempotent! + + def handle_event(event) + current_user_id = event.data[:current_user_id] + merge_request_id = event.data[:merge_request_id] + current_user = User.find_by_id(current_user_id) + + unless current_user + logger.info(structured_payload(message: 'Current user not found.', current_user_id: current_user_id)) + return + end + + merge_request = MergeRequest.find_by_id(merge_request_id) + + unless merge_request + logger.info(structured_payload(message: 'Merge request not found.', merge_request_id: merge_request_id)) + return + end + + TodoService.new.resolve_todos_for_target(merge_request, current_user) + end + end +end diff --git a/app/workers/new_issue_worker.rb b/app/workers/new_issue_worker.rb index 13936fac1e4..e14f0dc7dfe 100644 --- a/app/workers/new_issue_worker.rb +++ b/app/workers/new_issue_worker.rb @@ -13,7 +13,11 @@ class NewIssueWorker # rubocop:disable Scalability/IdempotentWorker worker_resource_boundary :cpu weight 2 - def perform(issue_id, user_id) + attr_reader :issuable_class + + def perform(issue_id, user_id, issuable_class = 'Issue') + @issuable_class = issuable_class.constantize + return unless objects_found?(issue_id, user_id) ::EventCreateService.new.open_issue(issuable, user) @@ -25,8 +29,4 @@ class NewIssueWorker # rubocop:disable Scalability/IdempotentWorker .new(project: issuable.project, current_user: user) .execute(issuable) end - - def issuable_class - Issue - end end diff --git a/app/workers/pages/invalidate_domain_cache_worker.rb b/app/workers/pages/invalidate_domain_cache_worker.rb index 63b6f5c05b5..97e8966b342 100644 --- a/app/workers/pages/invalidate_domain_cache_worker.rb +++ b/app/workers/pages/invalidate_domain_cache_worker.rb @@ -15,9 +15,13 @@ module Pages .clear_cache end - if event.data[:root_namespace_id] + event.data.values_at( + :root_namespace_id, + :old_root_namespace_id, + :new_root_namespace_id + ).compact.uniq.each do |namespace_id| ::Gitlab::Pages::CacheControl - .for_namespace(event.data[:root_namespace_id]) + .for_namespace(namespace_id) .clear_cache end end diff --git a/app/workers/post_receive.rb b/app/workers/post_receive.rb index 68a0934e2b7..329ccfc6362 100644 --- a/app/workers/post_receive.rb +++ b/app/workers/post_receive.rb @@ -85,6 +85,7 @@ class PostReceive replicate_snippet_changes(snippet) expire_caches(post_received, snippet.repository) + snippet.touch Snippets::UpdateStatisticsService.new(snippet).execute end diff --git a/app/workers/project_cache_worker.rb b/app/workers/project_cache_worker.rb index 0e90b41e28d..cb1a7c8560a 100644 --- a/app/workers/project_cache_worker.rb +++ b/app/workers/project_cache_worker.rb @@ -47,7 +47,8 @@ class ProjectCacheWorker Projects::UpdateStatisticsService.new(project, nil, statistics: statistics).execute - UpdateProjectStatisticsWorker.perform_in(LEASE_TIMEOUT, project.id, statistics) + lease_key = project_cache_worker_key(project.id, statistics) + UpdateProjectStatisticsWorker.perform_in(LEASE_TIMEOUT, lease_key, project.id, statistics) end private diff --git a/app/workers/projects/import_export/relation_export_worker.rb b/app/workers/projects/import_export/relation_export_worker.rb new file mode 100644 index 00000000000..13ca33c4457 --- /dev/null +++ b/app/workers/projects/import_export/relation_export_worker.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +module Projects + module ImportExport + class RelationExportWorker + include ApplicationWorker + include ExceptionBacktrace + + idempotent! + data_consistency :always + deduplicate :until_executed + feature_category :importers + sidekiq_options status_expiration: StuckExportJobsWorker::EXPORT_JOBS_EXPIRATION + urgency :low + worker_resource_boundary :memory + + def perform(project_relation_export_id) + relation_export = Projects::ImportExport::RelationExport.find(project_relation_export_id) + + if relation_export.queued? + Projects::ImportExport::RelationExportService.new(relation_export, jid).execute + end + end + end + end +end diff --git a/app/workers/service_desk_email_receiver_worker.rb b/app/workers/service_desk_email_receiver_worker.rb index c8ab8891856..b3b36ca2ada 100644 --- a/app/workers/service_desk_email_receiver_worker.rb +++ b/app/workers/service_desk_email_receiver_worker.rb @@ -9,9 +9,6 @@ class ServiceDeskEmailReceiverWorker < EmailReceiverWorker # rubocop:disable Sca urgency :high sidekiq_options retry: 3 - # https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/1263 - tags :needs_own_queue - def should_perform? ::Gitlab::ServiceDeskEmail.enabled? end diff --git a/app/workers/update_project_statistics_worker.rb b/app/workers/update_project_statistics_worker.rb index 45a6cc8f476..3308fa149f5 100644 --- a/app/workers/update_project_statistics_worker.rb +++ b/app/workers/update_project_statistics_worker.rb @@ -10,10 +10,15 @@ class UpdateProjectStatisticsWorker # rubocop:disable Scalability/IdempotentWork feature_category :source_code_management - # project_id - The ID of the project for which to flush the cache. - # statistics - An Array containing columns from ProjectStatistics to - # refresh, if empty all columns will be refreshed - def perform(project_id, statistics = []) + # lease_key - The exclusive lease key to take + # project_id - The ID of the project for which to flush the cache. + # statistics - An Array containing columns from ProjectStatistics to + # refresh, if empty all columns will be refreshed + def perform(lease_key, project_id, statistics = []) + return unless Gitlab::ExclusiveLease + .new(lease_key, timeout: ProjectCacheWorker::LEASE_TIMEOUT) + .try_obtain + project = Project.find_by_id(project_id) Projects::UpdateStatisticsService.new(project, nil, statistics: statistics).execute diff --git a/app/workers/users/deactivate_dormant_users_worker.rb b/app/workers/users/deactivate_dormant_users_worker.rb index d7ea20e4b62..b14b7e67450 100644 --- a/app/workers/users/deactivate_dormant_users_worker.rb +++ b/app/workers/users/deactivate_dormant_users_worker.rb @@ -10,43 +10,23 @@ module Users feature_category :utilization - NUMBER_OF_BATCHES = 50 - BATCH_SIZE = 200 - PAUSE_SECONDS = 0.25 - def perform return if Gitlab.com? return unless ::Gitlab::CurrentSettings.current_application_settings.deactivate_dormant_users - with_context(caller_id: self.class.name.to_s) do - NUMBER_OF_BATCHES.times do - result = User.connection.execute(update_query) - - break if result.cmd_tuples == 0 - - sleep(PAUSE_SECONDS) - end - end + deactivate_users(User.dormant) + deactivate_users(User.with_no_activity) end private - def update_query - <<~SQL - UPDATE "users" - SET "state" = 'deactivated' - WHERE "users"."id" IN ( - (#{users.dormant.to_sql}) - UNION - (#{users.with_no_activity.to_sql}) - LIMIT #{BATCH_SIZE} - ) - SQL - end - - def users - User.select(:id).limit(BATCH_SIZE) + def deactivate_users(scope) + with_context(caller_id: self.class.name.to_s) do + scope.each_batch do |batch| + batch.each(&:deactivate) + end + end end end end diff --git a/app/workers/x509_issuer_crl_check_worker.rb b/app/workers/x509_issuer_crl_check_worker.rb index 39440504927..cb5bae7ca4e 100644 --- a/app/workers/x509_issuer_crl_check_worker.rb +++ b/app/workers/x509_issuer_crl_check_worker.rb @@ -41,13 +41,13 @@ class X509IssuerCrlCheckWorker certs.find_each do |cert| logger.info(message: "Certificate revoked", - id: cert.id, - email: cert.email, - subject: cert.subject, - serial_number: cert.serial_number, - issuer: cert.x509_issuer.id, - issuer_subject: cert.x509_issuer.subject, - issuer_crl_url: cert.x509_issuer.crl_url) + id: cert.id, + email: cert.email, + subject: cert.subject, + serial_number: cert.serial_number, + issuer: cert.x509_issuer.id, + issuer_subject: cert.x509_issuer.subject, + issuer_crl_url: cert.x509_issuer.crl_url) end certs.update_all(certificate_status: :revoked) @@ -61,18 +61,18 @@ class X509IssuerCrlCheckWorker OpenSSL::X509::CRL.new(response.body) else logger.warn(message: "Failed to download certificate revocation list", - issuer: issuer.id, - issuer_subject: issuer.subject, - issuer_crl_url: issuer.crl_url) + issuer: issuer.id, + issuer_subject: issuer.subject, + issuer_crl_url: issuer.crl_url) nil end rescue OpenSSL::X509::CRLError logger.warn(message: "Failed to parse certificate revocation list", - issuer: issuer.id, - issuer_subject: issuer.subject, - issuer_crl_url: issuer.crl_url) + issuer: issuer.id, + issuer_subject: issuer.subject, + issuer_crl_url: issuer.crl_url) nil end |