Add latest changes from gitlab-org/gitlab@13-12-stable-eev13.12.0-rc42

1 files changed, 63 insertions, 1 deletions

diff --git a/app/services/web_hook_service.rb b/app/services/web_hook_service.rb
index 5a51b42f9f9..654d9356739 100644
--- a/app/services/web_hook_service.rb
+++ b/app/services/web_hook_service.rb
@@ -6,6 +6,18 @@ class WebHookService
 
     attr_reader :body, :headers, :code
 
+    def success?
+      false
+    end
+
+    def redirection?
+      false
+    end
+
+    def internal_server_error?
+      true
+    end
+
     def initialize
       @headers = Gitlab::HTTP::Response::Headers.new({})
       @body = ''
@@ -15,6 +27,7 @@ class WebHookService
 
   REQUEST_BODY_SIZE_LIMIT = 25.megabytes
   GITLAB_EVENT_HEADER = 'X-Gitlab-Event'
+  MAX_FAILURES = 100
 
   attr_accessor :hook, :data, :hook_name, :request_options
 
@@ -33,6 +46,8 @@ class WebHookService
   end
 
   def execute
+    return { status: :error, message: 'Hook disabled' } unless hook.executable?
+
     start_time = Gitlab::Metrics::System.monotonic_time
 
     response = if parsed_url.userinfo.blank?
@@ -76,7 +91,11 @@ class WebHookService
   end
 
   def async_execute
-    WebHookWorker.perform_async(hook.id, data, hook_name)
+    if rate_limited?(hook)
+      log_rate_limit(hook)
+    else
+      WebHookWorker.perform_async(hook.id, data, hook_name)
+    end
   end
 
   private
@@ -104,6 +123,8 @@ class WebHookService
   end
 
   def log_execution(trigger:, url:, request_data:, response:, execution_duration:, error_message: nil)
+    handle_failure(response, hook)
+
     WebHookLog.create(
       web_hook: hook,
       trigger: trigger,
@@ -118,6 +139,17 @@ class WebHookService
     )
   end
 
+  def handle_failure(response, hook)
+    if response.success? || response.redirection?
+      hook.enable!
+    elsif response.internal_server_error?
+      next_backoff = hook.next_backoff
+      hook.update!(disabled_until: next_backoff.from_now, backoff_count: hook.backoff_count + 1)
+    else
+      hook.update!(recent_failures: hook.recent_failures + 1) if hook.recent_failures < MAX_FAILURES
+    end
+  end
+
   def build_headers(hook_name)
     @headers ||= begin
       {
@@ -142,4 +174,34 @@ class WebHookService
 
     response.body.encode('UTF-8', invalid: :replace, undef: :replace, replace: '')
   end
+
+  def rate_limited?(hook)
+    return false unless Feature.enabled?(:web_hooks_rate_limit, default_enabled: :yaml)
+    return false if rate_limit.nil?
+
+    Gitlab::ApplicationRateLimiter.throttled?(
+      :web_hook_calls,
+      scope: [hook],
+      threshold: rate_limit
+    )
+  end
+
+  def rate_limit
+    @rate_limit ||= hook.rate_limit
+  end
+
+  def log_rate_limit(hook)
+    payload = {
+      message: 'Webhook rate limit exceeded',
+      hook_id: hook.id,
+      hook_type: hook.type,
+      hook_name: hook_name
+    }
+
+    Gitlab::AuthLogger.error(payload)
+
+    # Also log into application log for now, so we can use this information
+    # to determine suitable limits for gitlab.com
+    Gitlab::AppLogger.error(payload)
+  end
 end