Add latest changes from gitlab-org/gitlab@master

3 files changed, 30 insertions, 0 deletions

diff --git a/lib/gitlab/sidekiq_middleware/duplicate_jobs.rb b/lib/gitlab/sidekiq_middleware/duplicate_jobs.rb
new file mode 100644
index 00000000000..23222430902
--- /dev/null
+++ b/lib/gitlab/sidekiq_middleware/duplicate_jobs.rb
@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+require 'digest'
+
+module Gitlab
+  module SidekiqMiddleware
+    module DuplicateJobs
+      def self.drop_duplicates?
+        Feature.enabled?(:drop_duplicate_sidekiq_jobs)
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb b/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb
index b84673c4cee..c6fb50b4610 100644
--- a/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb
+++ b/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb
@@ -66,6 +66,10 @@ module Gitlab
           jid != existing_jid
         end
 
+        def droppable?
+          idempotent? && duplicate? && DuplicateJobs.drop_duplicates?
+        end
+
         private
 
         attr_reader :queue_name, :strategy, :job
@@ -98,6 +102,14 @@ module Gitlab
         def idempotency_string
           "#{worker_class_name}:#{arguments.join('-')}"
         end
+
+        def idempotent?
+          worker_class = worker_class_name.to_s.safe_constantize
+          return false unless worker_class
+          return false unless worker_class.respond_to?(:idempotent?)
+
+          worker_class.idempotent?
+        end
       end
     end
   end
diff --git a/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/until_executing.rb b/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/until_executing.rb
index b8f49b67a59..674e436b714 100644
--- a/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/until_executing.rb
+++ b/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/until_executing.rb
@@ -17,6 +17,11 @@ module Gitlab
               job['duplicate-of'] = duplicate_job.existing_jid
             end
 
+            if duplicate_job.droppable?
+              Gitlab::SidekiqLogging::DeduplicationLogger.instance.log(job, "dropped until executing")
+              return false
+            end
+
             yield
           end