diff options
Diffstat (limited to 'lib/gitlab/sidekiq_middleware')
5 files changed, 66 insertions, 25 deletions
diff --git a/lib/gitlab/sidekiq_middleware/admin_mode/client.rb b/lib/gitlab/sidekiq_middleware/admin_mode/client.rb index 36204e1bee0..1b33743a0e9 100644 --- a/lib/gitlab/sidekiq_middleware/admin_mode/client.rb +++ b/lib/gitlab/sidekiq_middleware/admin_mode/client.rb @@ -8,7 +8,8 @@ module Gitlab # If enabled then it injects a job field that persists through the job execution class Client def call(_worker_class, job, _queue, _redis_pool) - return yield unless ::Feature.enabled?(:user_mode_in_session) + # Not calling Gitlab::CurrentSettings.admin_mode on purpose on sidekiq middleware + # Only when admin mode application setting is enabled might the admin_mode_user_id be non-nil here # Admin mode enabled in the original request or in a nested sidekiq job admin_mode_user_id = find_admin_user_id diff --git a/lib/gitlab/sidekiq_middleware/admin_mode/server.rb b/lib/gitlab/sidekiq_middleware/admin_mode/server.rb index 6366867a0fa..c4e64705d6e 100644 --- a/lib/gitlab/sidekiq_middleware/admin_mode/server.rb +++ b/lib/gitlab/sidekiq_middleware/admin_mode/server.rb @@ -5,7 +5,8 @@ module Gitlab module AdminMode class Server def call(_worker, job, _queue) - return yield unless Feature.enabled?(:user_mode_in_session) + # Not calling Gitlab::CurrentSettings.admin_mode on purpose on sidekiq middleware + # Only when admin_mode setting is enabled can it be true here admin_mode_user_id = job['admin_mode_user_id'] diff --git a/lib/gitlab/sidekiq_middleware/instrumentation_logger.rb b/lib/gitlab/sidekiq_middleware/instrumentation_logger.rb index a66a4de4655..b542aa4fe4c 100644 --- a/lib/gitlab/sidekiq_middleware/instrumentation_logger.rb +++ b/lib/gitlab/sidekiq_middleware/instrumentation_logger.rb @@ -3,6 +3,24 @@ module Gitlab module SidekiqMiddleware class InstrumentationLogger + def self.keys + @keys ||= [ + :cpu_s, + :gitaly_calls, + :gitaly_duration_s, + :rugged_calls, + :rugged_duration_s, + :elasticsearch_calls, + :elasticsearch_duration_s, + :elasticsearch_timed_out_count, + *::Gitlab::Memory::Instrumentation::KEY_MAPPING.values, + *::Gitlab::Instrumentation::Redis.known_payload_keys, + *::Gitlab::Metrics::Subscribers::ActiveRecord.known_payload_keys, + *::Gitlab::Metrics::Subscribers::ExternalHttp::KNOWN_PAYLOAD_KEYS, + *::Gitlab::Metrics::Subscribers::RackAttack::PAYLOAD_KEYS + ] + end + def call(worker, job, queue) ::Gitlab::InstrumentationHelper.init_instrumentation_data @@ -17,7 +35,10 @@ module Gitlab # because Sidekiq keeps a pristine copy of the original hash # before sending it to the middleware: # https://github.com/mperham/sidekiq/blob/53bd529a0c3f901879925b8390353129c465b1f2/lib/sidekiq/processor.rb#L115-L118 - ::Gitlab::InstrumentationHelper.add_instrumentation_data(job) + job[:instrumentation] = {}.tap do |instrumentation_values| + ::Gitlab::InstrumentationHelper.add_instrumentation_data(instrumentation_values) + instrumentation_values.slice!(*self.class.keys) + end end end end diff --git a/lib/gitlab/sidekiq_middleware/metrics_helper.rb b/lib/gitlab/sidekiq_middleware/metrics_helper.rb index 60e79ee1188..66930a34319 100644 --- a/lib/gitlab/sidekiq_middleware/metrics_helper.rb +++ b/lib/gitlab/sidekiq_middleware/metrics_helper.rb @@ -10,6 +10,7 @@ module Gitlab def create_labels(worker_class, queue, job) worker_name = (job['wrapped'].presence || worker_class).to_s + worker = find_worker(worker_name, worker_class) labels = { queue: queue.to_s, worker: worker_name, @@ -18,15 +19,15 @@ module Gitlab feature_category: "", boundary: "" } - return labels unless worker_class && worker_class.include?(WorkerAttributes) + return labels unless worker.respond_to?(:get_urgency) - labels[:urgency] = worker_class.get_urgency.to_s - labels[:external_dependencies] = bool_as_label(worker_class.worker_has_external_dependencies?) + labels[:urgency] = worker.get_urgency.to_s + labels[:external_dependencies] = bool_as_label(worker.worker_has_external_dependencies?) - feature_category = worker_class.get_feature_category + feature_category = worker.get_feature_category labels[:feature_category] = feature_category.to_s - resource_boundary = worker_class.get_worker_resource_boundary + resource_boundary = worker.get_worker_resource_boundary labels[:boundary] = resource_boundary == :unknown ? "" : resource_boundary.to_s labels @@ -35,6 +36,10 @@ module Gitlab def bool_as_label(value) value ? TRUE_LABEL : FALSE_LABEL end + + def find_worker(worker_name, worker_class) + Gitlab::SidekiqConfig::DEFAULT_WORKERS.fetch(worker_name, worker_class) + end end end end diff --git a/lib/gitlab/sidekiq_middleware/server_metrics.rb b/lib/gitlab/sidekiq_middleware/server_metrics.rb index cf768811ffd..f5fee8050ac 100644 --- a/lib/gitlab/sidekiq_middleware/server_metrics.rb +++ b/lib/gitlab/sidekiq_middleware/server_metrics.rb @@ -21,6 +21,16 @@ module Gitlab Thread.current.name ||= Gitlab::Metrics::Samplers::ThreadsSampler::SIDEKIQ_WORKER_THREAD_NAME labels = create_labels(worker.class, queue, job) + instrument(job, labels) do + yield + end + end + + protected + + attr_reader :metrics + + def instrument(job, labels) queue_duration = ::Gitlab::InstrumentationHelper.queue_duration_for_job(job) @metrics[:sidekiq_jobs_queue_duration_seconds].observe(labels, queue_duration) if queue_duration @@ -50,19 +60,18 @@ module Gitlab # job_status: done, fail match the job_status attribute in structured logging labels[:job_status] = job_succeeded ? "done" : "fail" + instrumentation = job[:instrumentation] || {} @metrics[:sidekiq_jobs_cpu_seconds].observe(labels, job_thread_cputime) @metrics[:sidekiq_jobs_completion_seconds].observe(labels, monotonic_time) @metrics[:sidekiq_jobs_db_seconds].observe(labels, ActiveRecord::LogSubscriber.runtime / 1000) - @metrics[:sidekiq_jobs_gitaly_seconds].observe(labels, get_gitaly_time(job)) - @metrics[:sidekiq_redis_requests_total].increment(labels, get_redis_calls(job)) - @metrics[:sidekiq_redis_requests_duration_seconds].observe(labels, get_redis_time(job)) - @metrics[:sidekiq_elasticsearch_requests_total].increment(labels, get_elasticsearch_calls(job)) - @metrics[:sidekiq_elasticsearch_requests_duration_seconds].observe(labels, get_elasticsearch_time(job)) + @metrics[:sidekiq_jobs_gitaly_seconds].observe(labels, get_gitaly_time(instrumentation)) + @metrics[:sidekiq_redis_requests_total].increment(labels, get_redis_calls(instrumentation)) + @metrics[:sidekiq_redis_requests_duration_seconds].observe(labels, get_redis_time(instrumentation)) + @metrics[:sidekiq_elasticsearch_requests_total].increment(labels, get_elasticsearch_calls(instrumentation)) + @metrics[:sidekiq_elasticsearch_requests_duration_seconds].observe(labels, get_elasticsearch_time(instrumentation)) end end - private - def init_metrics { sidekiq_jobs_cpu_seconds: ::Gitlab::Metrics.histogram(:sidekiq_jobs_cpu_seconds, 'Seconds of cpu time to run Sidekiq job', {}, SIDEKIQ_LATENCY_BUCKETS), @@ -81,29 +90,33 @@ module Gitlab } end + private + def get_thread_cputime defined?(Process::CLOCK_THREAD_CPUTIME_ID) ? Process.clock_gettime(Process::CLOCK_THREAD_CPUTIME_ID) : 0 end - def get_redis_time(job) - job.fetch(:redis_duration_s, 0) + def get_redis_time(payload) + payload.fetch(:redis_duration_s, 0) end - def get_redis_calls(job) - job.fetch(:redis_calls, 0) + def get_redis_calls(payload) + payload.fetch(:redis_calls, 0) end - def get_elasticsearch_time(job) - job.fetch(:elasticsearch_duration_s, 0) + def get_elasticsearch_time(payload) + payload.fetch(:elasticsearch_duration_s, 0) end - def get_elasticsearch_calls(job) - job.fetch(:elasticsearch_calls, 0) + def get_elasticsearch_calls(payload) + payload.fetch(:elasticsearch_calls, 0) end - def get_gitaly_time(job) - job.fetch(:gitaly_duration_s, 0) + def get_gitaly_time(payload) + payload.fetch(:gitaly_duration_s, 0) end end end end + +Gitlab::SidekiqMiddleware::ServerMetrics.prepend_if_ee('EE::Gitlab::SidekiqMiddleware::ServerMetrics') |