Add latest changes from gitlab-org/gitlab@14-0-stable-eev14.0.0-rc42

3 files changed, 44 insertions, 2 deletions

diff --git a/app/workers/concerns/application_worker.rb b/app/workers/concerns/application_worker.rb
index 843be4896a3..3cba1eb31c5 100644
--- a/app/workers/concerns/application_worker.rb
+++ b/app/workers/concerns/application_worker.rb
@@ -13,6 +13,7 @@ module ApplicationWorker
   include Gitlab::SidekiqVersioning::Worker
 
   LOGGING_EXTRA_KEY = 'extra'
+  DEFAULT_DELAY_INTERVAL = 1
 
   included do
     set_queue
@@ -51,6 +52,16 @@ module ApplicationWorker
       subclass.after_set_class_attribute { subclass.set_queue }
     end
 
+    def perform_async(*args)
+      # Worker execution for workers with data_consistency set to :delayed or :sticky
+      # will be delayed to give replication enough time to complete
+      if utilizes_load_balancing_capabilities?
+        perform_in(delay_interval, *args)
+      else
+        super
+      end
+    end
+
     def set_queue
       queue_name = ::Gitlab::SidekiqConfig::WorkerRouter.global.route(self)
       sidekiq_options queue: queue_name # rubocop:disable Cop/SidekiqOptionsQueue
@@ -111,5 +122,11 @@ module ApplicationWorker
         Sidekiq::Client.push_bulk('class' => self, 'args' => args_list, 'at' => schedule)
       end
     end
+
+    protected
+
+    def delay_interval
+      DEFAULT_DELAY_INTERVAL.seconds
+    end
   end
 end
diff --git a/app/workers/concerns/security_scans_queue.rb b/app/workers/concerns/security_scans_queue.rb
index f731317bb37..27e97169926 100644
--- a/app/workers/concerns/security_scans_queue.rb
+++ b/app/workers/concerns/security_scans_queue.rb
@@ -8,6 +8,6 @@ module SecurityScansQueue
 
   included do
     queue_namespace :security_scans
-    feature_category :static_application_security_testing
+    feature_category :vulnerability_management
   end
 end
diff --git a/app/workers/concerns/worker_attributes.rb b/app/workers/concerns/worker_attributes.rb
index 6dee9402691..096be808787 100644
--- a/app/workers/concerns/worker_attributes.rb
+++ b/app/workers/concerns/worker_attributes.rb
@@ -71,6 +71,20 @@ module WorkerAttributes
       class_attributes[:urgency] || :low
     end
 
+    # Allows configuring worker's data_consistency.
+    #
+    #  Worker can utilize Sidekiq readonly database replicas capabilities by setting data_consistency attribute.
+    #  Workers with data_consistency set to :delayed or :sticky, calling #perform_async
+    #  will be delayed in order to give replication process enough time to complete.
+    #
+    #  - *data_consistency* values:
+    #    - 'always' - The job is required to use the primary database (default).
+    #    - 'sticky' - The uses a replica as long as possible. It switches to primary either on write or long replication lag.
+    #    - 'delayed' - The job would switch to primary only on write. It would use replica always.
+    #      If there's a long replication lag the job will be delayed, and only if the replica is not up to date on the next retry,
+    #      it will switch to the primary.
+    #  - *feature_flag* - allows you to toggle a job's `data_consistency, which permits you to safely toggle load balancing capabilities for a specific job.
+    #    If disabled, job will default to `:always`, which means that the job will always use the primary.
     def data_consistency(data_consistency, feature_flag: nil)
       raise ArgumentError, "Invalid data consistency: #{data_consistency}" unless VALID_DATA_CONSISTENCIES.include?(data_consistency)
       raise ArgumentError, 'Data consistency is already set' if class_attributes[:data_consistency]
@@ -85,11 +99,16 @@ module WorkerAttributes
       # Since the deduplication should always take into account the latest binary replication pointer into account,
       # not the first one, the deduplication will not work with sticky or delayed.
       # Follow up issue to improve this: https://gitlab.com/gitlab-org/gitlab/-/issues/325291
-      if idempotent? && get_data_consistency != :always
+      if idempotent? && utilizes_load_balancing_capabilities?
         raise ArgumentError, "Class can't be marked as idempotent if data_consistency is not set to :always"
       end
     end
 
+    # If data_consistency is not set to :always, worker will try to utilize load balancing capabilities and use the replica
+    def utilizes_load_balancing_capabilities?
+      get_data_consistency != :always
+    end
+
     def get_data_consistency
       class_attributes[:data_consistency] || :always
     end
@@ -167,6 +186,12 @@ module WorkerAttributes
       class_attributes[:deduplication_options] || {}
     end
 
+    def deduplication_enabled?
+      return true unless get_deduplication_options[:feature_flag]
+
+      Feature.enabled?(get_deduplication_options[:feature_flag], default_enabled: :yaml)
+    end
+
     def big_payload!
       set_class_attribute(:big_payload, true)
     end