Add latest changes from gitlab-org/gitlab@13-10-stable-eev13.10.0-rc40

4 files changed, 147 insertions, 1 deletions

diff --git a/lib/gitlab/sidekiq_middleware/server_metrics.rb b/lib/gitlab/sidekiq_middleware/server_metrics.rb
index 4ab8d313ad8..cf768811ffd 100644
--- a/lib/gitlab/sidekiq_middleware/server_metrics.rb
+++ b/lib/gitlab/sidekiq_middleware/server_metrics.rb
@@ -34,7 +34,8 @@ module Gitlab
         monotonic_time_start = Gitlab::Metrics::System.monotonic_time
         job_thread_cputime_start = get_thread_cputime
         begin
-          yield
+          transaction = Gitlab::Metrics::BackgroundTransaction.new
+          transaction.run { yield }
           job_succeeded = true
         ensure
           monotonic_time_end = Gitlab::Metrics::System.monotonic_time
diff --git a/lib/gitlab/sidekiq_middleware/size_limiter/client.rb b/lib/gitlab/sidekiq_middleware/size_limiter/client.rb
new file mode 100644
index 00000000000..bc8b1989e78
--- /dev/null
+++ b/lib/gitlab/sidekiq_middleware/size_limiter/client.rb
@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module SidekiqMiddleware
+    module SizeLimiter
+      # This midleware is inserted into Sidekiq **client** middleware chain. It
+      # prevents the caller from dispatching a too-large job payload. The job
+      # payload should be small and simple. Read more at:
+      # https://github.com/mperham/sidekiq/wiki/Best-Practices#1-make-your-job-parameters-small-and-simple
+      class Client
+        def call(worker_class, job, queue, _redis_pool)
+          ::Gitlab::SidekiqMiddleware::SizeLimiter::Validator.validate!(worker_class, job)
+
+          yield
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/sidekiq_middleware/size_limiter/exceed_limit_error.rb b/lib/gitlab/sidekiq_middleware/size_limiter/exceed_limit_error.rb
new file mode 100644
index 00000000000..da6c903ccae
--- /dev/null
+++ b/lib/gitlab/sidekiq_middleware/size_limiter/exceed_limit_error.rb
@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module SidekiqMiddleware
+    module SizeLimiter
+      # A custom exception for size limiter. It contains worker class and its
+      # size to easier track later
+      class ExceedLimitError < StandardError
+        attr_reader :worker_class, :size, :size_limit
+
+        def initialize(worker_class, size, size_limit)
+          @worker_class = worker_class
+          @size = size
+          @size_limit = size_limit
+
+          super "#{@worker_class} job exceeds payload size limit (#{size}/#{size_limit})"
+        end
+
+        def sentry_extra_data
+          {
+            worker_class: @worker_class.to_s,
+            size: @size.to_i,
+            size_limit: @size_limit.to_i
+          }
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/sidekiq_middleware/size_limiter/validator.rb b/lib/gitlab/sidekiq_middleware/size_limiter/validator.rb
new file mode 100644
index 00000000000..2c50c4a2157
--- /dev/null
+++ b/lib/gitlab/sidekiq_middleware/size_limiter/validator.rb
@@ -0,0 +1,97 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module SidekiqMiddleware
+    module SizeLimiter
+      # Validate a Sidekiq job payload limit based on current configuration.
+      # This validator pulls the configuration from the environment variables:
+      #
+      # - GITLAB_SIDEKIQ_SIZE_LIMITER_MODE: the current mode of the size
+      # limiter. This must be either `track` or `raise`.
+      #
+      # - GITLAB_SIDEKIQ_SIZE_LIMITER_LIMIT_BYTES: the size limit in bytes.
+      #
+      # If the size of job payload after serialization exceeds the limit, an
+      # error is tracked raised adhering to the mode.
+      class Validator
+        def self.validate!(worker_class, job)
+          new(worker_class, job).validate!
+        end
+
+        DEFAULT_SIZE_LIMIT = 0
+
+        MODES = [
+          TRACK_MODE = 'track',
+          RAISE_MODE = 'raise'
+        ].freeze
+
+        attr_reader :mode, :size_limit
+
+        def initialize(
+          worker_class, job,
+          mode: ENV['GITLAB_SIDEKIQ_SIZE_LIMITER_MODE'],
+          size_limit: ENV['GITLAB_SIDEKIQ_SIZE_LIMITER_LIMIT_BYTES']
+        )
+          @worker_class = worker_class
+          @job = job
+
+          @mode = (mode || TRACK_MODE).to_s.strip
+          unless MODES.include?(@mode)
+            ::Sidekiq.logger.warn "Invalid Sidekiq size limiter mode: #{@mode}. Fallback to #{TRACK_MODE} mode."
+            @mode = TRACK_MODE
+          end
+
+          @size_limit = (size_limit || DEFAULT_SIZE_LIMIT).to_i
+          if @size_limit < 0
+            ::Sidekiq.logger.warn "Invalid Sidekiq size limiter limit: #{@size_limit}"
+          end
+        end
+
+        def validate!
+          return unless @size_limit > 0
+
+          return if allow_big_payload?
+          return if job_size <= @size_limit
+
+          exception = ExceedLimitError.new(@worker_class, job_size, @size_limit)
+          # This should belong to Gitlab::ErrorTracking. We'll remove this
+          # after this epic is done:
+          # https://gitlab.com/groups/gitlab-com/gl-infra/-/epics/396
+          exception.set_backtrace(backtrace)
+
+          if raise_mode?
+            raise exception
+          else
+            track(exception)
+          end
+        end
+
+        private
+
+        def job_size
+          # This maynot be the optimal solution, but can be acceptable solution
+          # for now. Internally, Sidekiq calls Sidekiq.dump_json everywhere.
+          # There is no clean way to intefere to prevent double serialization.
+          @job_size ||= ::Sidekiq.dump_json(@job).bytesize
+        end
+
+        def allow_big_payload?
+          worker_class = @worker_class.to_s.safe_constantize
+          worker_class.respond_to?(:big_payload?) && worker_class.big_payload?
+        end
+
+        def raise_mode?
+          @mode == RAISE_MODE
+        end
+
+        def track(exception)
+          Gitlab::ErrorTracking.track_exception(exception)
+        end
+
+        def backtrace
+          Gitlab::BacktraceCleaner.clean_backtrace(caller)
+        end
+      end
+    end
+  end
+end