1 files changed, 75 insertions, 7 deletions

diff --git a/app/models/concerns/web_hooks/auto_disabling.rb b/app/models/concerns/web_hooks/auto_disabling.rb
index 2cc17a6f185..5f9901901d4 100644
--- a/app/models/concerns/web_hooks/auto_disabling.rb
+++ b/app/models/concerns/web_hooks/auto_disabling.rb
@@ -4,7 +4,20 @@ module WebHooks
   module AutoDisabling
     extend ActiveSupport::Concern
 
+    ENABLED_HOOK_TYPES = %w[ProjectHook].freeze
+
+    class_methods do
+      def auto_disabling_enabled?
+        ENABLED_HOOK_TYPES.include?(name) &&
+          Gitlab::SafeRequestStore.fetch(:auto_disabling_web_hooks) do
+            Feature.enabled?(:auto_disabling_web_hooks, type: :ops)
+          end
+      end
+    end
+
     included do
+      delegate :auto_disabling_enabled?, to: :class, private: true
+
       # A hook is disabled if:
       #
       # - we are no longer in the grace-perod (recent_failures > ?)
@@ -12,6 +25,8 @@ module WebHooks
       #   - disabled_until is nil (i.e. this was set by WebHook#fail!)
       #   - or disabled_until is in the future (i.e. this was set by WebHook#backoff!)
       scope :disabled, -> do
+        return none unless auto_disabling_enabled?
+
         where('recent_failures > ? AND (disabled_until IS NULL OR disabled_until >= ?)',
               WebHook::FAILURE_THRESHOLD, Time.current)
       end
@@ -23,40 +38,83 @@ module WebHooks
       #   - disabled_until is nil (i.e. this was set by WebHook#fail!)
       #   - disabled_until is in the future (i.e. this was set by WebHook#backoff!)
       scope :executable, -> do
+        return all unless auto_disabling_enabled?
+
         where('recent_failures <= ? OR (recent_failures > ? AND (disabled_until IS NOT NULL) AND (disabled_until < ?))',
               WebHook::FAILURE_THRESHOLD, WebHook::FAILURE_THRESHOLD, Time.current)
       end
     end
 
     def executable?
+      return true unless auto_disabling_enabled?
+
       !temporarily_disabled? && !permanently_disabled?
     end
 
     def temporarily_disabled?
-      return false if recent_failures <= WebHook::FAILURE_THRESHOLD
+      return false unless auto_disabling_enabled?
 
-      disabled_until.present? && disabled_until >= Time.current
+      disabled_until.present? && disabled_until >= Time.current && recent_failures > WebHook::FAILURE_THRESHOLD
     end
 
     def permanently_disabled?
-      return false if disabled_until.present?
+      return false unless auto_disabling_enabled?
 
-      recent_failures > WebHook::FAILURE_THRESHOLD
+      recent_failures > WebHook::FAILURE_THRESHOLD && disabled_until.blank?
     end
 
     def disable!
-      return if permanently_disabled?
+      return if !auto_disabling_enabled? || permanently_disabled?
+
+      update_attribute(:recent_failures, WebHook::EXCEEDED_FAILURE_THRESHOLD)
+    end
+
+    def enable!
+      return unless auto_disabling_enabled?
+      return if recent_failures == 0 && disabled_until.nil? && backoff_count == 0
 
-      super
+      assign_attributes(recent_failures: 0, disabled_until: nil, backoff_count: 0)
+      save(validate: false)
     end
 
+    # Don't actually back-off until FAILURE_THRESHOLD failures have been seen
+    # we mark the grace-period using the recent_failures counter
     def backoff!
+      return unless auto_disabling_enabled?
       return if permanently_disabled? || (backoff_count >= WebHook::MAX_FAILURES && temporarily_disabled?)
 
-      super
+      attrs = { recent_failures: next_failure_count }
+
+      if recent_failures >= WebHook::FAILURE_THRESHOLD
+        attrs[:backoff_count] = next_backoff_count
+        attrs[:disabled_until] = next_backoff.from_now
+      end
+
+      assign_attributes(attrs)
+      save(validate: false) if changed?
+    end
+
+    def failed!
+      return unless auto_disabling_enabled?
+      return unless recent_failures < WebHook::MAX_FAILURES
+
+      assign_attributes(disabled_until: nil, backoff_count: 0, recent_failures: next_failure_count)
+      save(validate: false)
+    end
+
+    def next_backoff
+      if backoff_count >= 8 # optimization to prevent expensive exponentiation and possible overflows
+        return WebHook::MAX_BACKOFF
+      end
+
+      (WebHook::INITIAL_BACKOFF * (WebHook::BACKOFF_GROWTH_FACTOR**backoff_count))
+        .clamp(WebHook::INITIAL_BACKOFF, WebHook::MAX_BACKOFF)
+        .seconds
     end
 
     def alert_status
+      return :executable unless auto_disabling_enabled?
+
       if temporarily_disabled?
         :temporarily_disabled
       elsif permanently_disabled?
@@ -65,5 +123,15 @@ module WebHooks
         :executable
       end
     end
+
+    private
+
+    def next_failure_count
+      recent_failures.succ.clamp(1, WebHook::MAX_FAILURES)
+    end
+
+    def next_backoff_count
+      backoff_count.succ.clamp(1, WebHook::MAX_FAILURES)
+    end
   end
 end