diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2021-03-16 21:18:33 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2021-03-16 21:18:33 +0300 |
commit | f64a639bcfa1fc2bc89ca7db268f594306edfd7c (patch) | |
tree | a2c3c2ebcc3b45e596949db485d6ed18ffaacfa1 /lib/gitlab/sidekiq_middleware | |
parent | bfbc3e0d6583ea1a91f627528bedc3d65ba4b10f (diff) |
Add latest changes from gitlab-org/gitlab@13-10-stable-eev13.10.0-rc40
Diffstat (limited to 'lib/gitlab/sidekiq_middleware')
4 files changed, 147 insertions, 1 deletions
diff --git a/lib/gitlab/sidekiq_middleware/server_metrics.rb b/lib/gitlab/sidekiq_middleware/server_metrics.rb index 4ab8d313ad8..cf768811ffd 100644 --- a/lib/gitlab/sidekiq_middleware/server_metrics.rb +++ b/lib/gitlab/sidekiq_middleware/server_metrics.rb @@ -34,7 +34,8 @@ module Gitlab monotonic_time_start = Gitlab::Metrics::System.monotonic_time job_thread_cputime_start = get_thread_cputime begin - yield + transaction = Gitlab::Metrics::BackgroundTransaction.new + transaction.run { yield } job_succeeded = true ensure monotonic_time_end = Gitlab::Metrics::System.monotonic_time diff --git a/lib/gitlab/sidekiq_middleware/size_limiter/client.rb b/lib/gitlab/sidekiq_middleware/size_limiter/client.rb new file mode 100644 index 00000000000..bc8b1989e78 --- /dev/null +++ b/lib/gitlab/sidekiq_middleware/size_limiter/client.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module Gitlab + module SidekiqMiddleware + module SizeLimiter + # This midleware is inserted into Sidekiq **client** middleware chain. It + # prevents the caller from dispatching a too-large job payload. The job + # payload should be small and simple. Read more at: + # https://github.com/mperham/sidekiq/wiki/Best-Practices#1-make-your-job-parameters-small-and-simple + class Client + def call(worker_class, job, queue, _redis_pool) + ::Gitlab::SidekiqMiddleware::SizeLimiter::Validator.validate!(worker_class, job) + + yield + end + end + end + end +end diff --git a/lib/gitlab/sidekiq_middleware/size_limiter/exceed_limit_error.rb b/lib/gitlab/sidekiq_middleware/size_limiter/exceed_limit_error.rb new file mode 100644 index 00000000000..da6c903ccae --- /dev/null +++ b/lib/gitlab/sidekiq_middleware/size_limiter/exceed_limit_error.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +module Gitlab + module SidekiqMiddleware + module SizeLimiter + # A custom exception for size limiter. It contains worker class and its + # size to easier track later + class ExceedLimitError < StandardError + attr_reader :worker_class, :size, :size_limit + + def initialize(worker_class, size, size_limit) + @worker_class = worker_class + @size = size + @size_limit = size_limit + + super "#{@worker_class} job exceeds payload size limit (#{size}/#{size_limit})" + end + + def sentry_extra_data + { + worker_class: @worker_class.to_s, + size: @size.to_i, + size_limit: @size_limit.to_i + } + end + end + end + end +end diff --git a/lib/gitlab/sidekiq_middleware/size_limiter/validator.rb b/lib/gitlab/sidekiq_middleware/size_limiter/validator.rb new file mode 100644 index 00000000000..2c50c4a2157 --- /dev/null +++ b/lib/gitlab/sidekiq_middleware/size_limiter/validator.rb @@ -0,0 +1,97 @@ +# frozen_string_literal: true + +module Gitlab + module SidekiqMiddleware + module SizeLimiter + # Validate a Sidekiq job payload limit based on current configuration. + # This validator pulls the configuration from the environment variables: + # + # - GITLAB_SIDEKIQ_SIZE_LIMITER_MODE: the current mode of the size + # limiter. This must be either `track` or `raise`. + # + # - GITLAB_SIDEKIQ_SIZE_LIMITER_LIMIT_BYTES: the size limit in bytes. + # + # If the size of job payload after serialization exceeds the limit, an + # error is tracked raised adhering to the mode. + class Validator + def self.validate!(worker_class, job) + new(worker_class, job).validate! + end + + DEFAULT_SIZE_LIMIT = 0 + + MODES = [ + TRACK_MODE = 'track', + RAISE_MODE = 'raise' + ].freeze + + attr_reader :mode, :size_limit + + def initialize( + worker_class, job, + mode: ENV['GITLAB_SIDEKIQ_SIZE_LIMITER_MODE'], + size_limit: ENV['GITLAB_SIDEKIQ_SIZE_LIMITER_LIMIT_BYTES'] + ) + @worker_class = worker_class + @job = job + + @mode = (mode || TRACK_MODE).to_s.strip + unless MODES.include?(@mode) + ::Sidekiq.logger.warn "Invalid Sidekiq size limiter mode: #{@mode}. Fallback to #{TRACK_MODE} mode." + @mode = TRACK_MODE + end + + @size_limit = (size_limit || DEFAULT_SIZE_LIMIT).to_i + if @size_limit < 0 + ::Sidekiq.logger.warn "Invalid Sidekiq size limiter limit: #{@size_limit}" + end + end + + def validate! + return unless @size_limit > 0 + + return if allow_big_payload? + return if job_size <= @size_limit + + exception = ExceedLimitError.new(@worker_class, job_size, @size_limit) + # This should belong to Gitlab::ErrorTracking. We'll remove this + # after this epic is done: + # https://gitlab.com/groups/gitlab-com/gl-infra/-/epics/396 + exception.set_backtrace(backtrace) + + if raise_mode? + raise exception + else + track(exception) + end + end + + private + + def job_size + # This maynot be the optimal solution, but can be acceptable solution + # for now. Internally, Sidekiq calls Sidekiq.dump_json everywhere. + # There is no clean way to intefere to prevent double serialization. + @job_size ||= ::Sidekiq.dump_json(@job).bytesize + end + + def allow_big_payload? + worker_class = @worker_class.to_s.safe_constantize + worker_class.respond_to?(:big_payload?) && worker_class.big_payload? + end + + def raise_mode? + @mode == RAISE_MODE + end + + def track(exception) + Gitlab::ErrorTracking.track_exception(exception) + end + + def backtrace + Gitlab::BacktraceCleaner.clean_backtrace(caller) + end + end + end + end +end |