diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2021-06-16 21:25:58 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2021-06-16 21:25:58 +0300 |
commit | a5f4bba440d7f9ea47046a0a561d49adf0a1e6d4 (patch) | |
tree | fb69158581673816a8cd895f9d352dcb3c678b1e /app/workers/concerns | |
parent | d16b2e8639e99961de6ddc93909f3bb5c1445ba1 (diff) |
Add latest changes from gitlab-org/gitlab@14-0-stable-eev14.0.0-rc42
Diffstat (limited to 'app/workers/concerns')
-rw-r--r-- | app/workers/concerns/application_worker.rb | 17 | ||||
-rw-r--r-- | app/workers/concerns/security_scans_queue.rb | 2 | ||||
-rw-r--r-- | app/workers/concerns/worker_attributes.rb | 27 |
3 files changed, 44 insertions, 2 deletions
diff --git a/app/workers/concerns/application_worker.rb b/app/workers/concerns/application_worker.rb index 843be4896a3..3cba1eb31c5 100644 --- a/app/workers/concerns/application_worker.rb +++ b/app/workers/concerns/application_worker.rb @@ -13,6 +13,7 @@ module ApplicationWorker include Gitlab::SidekiqVersioning::Worker LOGGING_EXTRA_KEY = 'extra' + DEFAULT_DELAY_INTERVAL = 1 included do set_queue @@ -51,6 +52,16 @@ module ApplicationWorker subclass.after_set_class_attribute { subclass.set_queue } end + def perform_async(*args) + # Worker execution for workers with data_consistency set to :delayed or :sticky + # will be delayed to give replication enough time to complete + if utilizes_load_balancing_capabilities? + perform_in(delay_interval, *args) + else + super + end + end + def set_queue queue_name = ::Gitlab::SidekiqConfig::WorkerRouter.global.route(self) sidekiq_options queue: queue_name # rubocop:disable Cop/SidekiqOptionsQueue @@ -111,5 +122,11 @@ module ApplicationWorker Sidekiq::Client.push_bulk('class' => self, 'args' => args_list, 'at' => schedule) end end + + protected + + def delay_interval + DEFAULT_DELAY_INTERVAL.seconds + end end end diff --git a/app/workers/concerns/security_scans_queue.rb b/app/workers/concerns/security_scans_queue.rb index f731317bb37..27e97169926 100644 --- a/app/workers/concerns/security_scans_queue.rb +++ b/app/workers/concerns/security_scans_queue.rb @@ -8,6 +8,6 @@ module SecurityScansQueue included do queue_namespace :security_scans - feature_category :static_application_security_testing + feature_category :vulnerability_management end end diff --git a/app/workers/concerns/worker_attributes.rb b/app/workers/concerns/worker_attributes.rb index 6dee9402691..096be808787 100644 --- a/app/workers/concerns/worker_attributes.rb +++ b/app/workers/concerns/worker_attributes.rb @@ -71,6 +71,20 @@ module WorkerAttributes class_attributes[:urgency] || :low end + # Allows configuring worker's data_consistency. + # + # Worker can utilize Sidekiq readonly database replicas capabilities by setting data_consistency attribute. + # Workers with data_consistency set to :delayed or :sticky, calling #perform_async + # will be delayed in order to give replication process enough time to complete. + # + # - *data_consistency* values: + # - 'always' - The job is required to use the primary database (default). + # - 'sticky' - The uses a replica as long as possible. It switches to primary either on write or long replication lag. + # - 'delayed' - The job would switch to primary only on write. It would use replica always. + # If there's a long replication lag the job will be delayed, and only if the replica is not up to date on the next retry, + # it will switch to the primary. + # - *feature_flag* - allows you to toggle a job's `data_consistency, which permits you to safely toggle load balancing capabilities for a specific job. + # If disabled, job will default to `:always`, which means that the job will always use the primary. def data_consistency(data_consistency, feature_flag: nil) raise ArgumentError, "Invalid data consistency: #{data_consistency}" unless VALID_DATA_CONSISTENCIES.include?(data_consistency) raise ArgumentError, 'Data consistency is already set' if class_attributes[:data_consistency] @@ -85,11 +99,16 @@ module WorkerAttributes # Since the deduplication should always take into account the latest binary replication pointer into account, # not the first one, the deduplication will not work with sticky or delayed. # Follow up issue to improve this: https://gitlab.com/gitlab-org/gitlab/-/issues/325291 - if idempotent? && get_data_consistency != :always + if idempotent? && utilizes_load_balancing_capabilities? raise ArgumentError, "Class can't be marked as idempotent if data_consistency is not set to :always" end end + # If data_consistency is not set to :always, worker will try to utilize load balancing capabilities and use the replica + def utilizes_load_balancing_capabilities? + get_data_consistency != :always + end + def get_data_consistency class_attributes[:data_consistency] || :always end @@ -167,6 +186,12 @@ module WorkerAttributes class_attributes[:deduplication_options] || {} end + def deduplication_enabled? + return true unless get_deduplication_options[:feature_flag] + + Feature.enabled?(get_deduplication_options[:feature_flag], default_enabled: :yaml) + end + def big_payload! set_class_attribute(:big_payload, true) end |