diff options
author | Bob Van Landuyt <bob@gitlab.com> | 2019-06-04 20:59:04 +0300 |
---|---|---|
committer | Bob Van Landuyt <bob@gitlab.com> | 2019-06-04 20:59:04 +0300 |
commit | 327cae1da7f91256629fc59cfd4e4c97fd77366a (patch) | |
tree | 169c079dd305100e896eb437da69d1b0932cd146 /lib/gitlab/metrics/samplers | |
parent | 58b52e45178945cc770c78e64b970d54181bdda9 (diff) | |
parent | 4c6b1fc23c72dcde5456865103945ce7f228bfef (diff) |
Merge branch '61964-unicorn-instrumentation' into 'master'
Adds ruby and unicorn instrumentation
Closes #61964
See merge request gitlab-org/gitlab-ce!28499
Diffstat (limited to 'lib/gitlab/metrics/samplers')
-rw-r--r-- | lib/gitlab/metrics/samplers/ruby_sampler.rb | 31 | ||||
-rw-r--r-- | lib/gitlab/metrics/samplers/unicorn_sampler.rb | 36 |
2 files changed, 46 insertions, 21 deletions
diff --git a/lib/gitlab/metrics/samplers/ruby_sampler.rb b/lib/gitlab/metrics/samplers/ruby_sampler.rb index 18a69321905..4d9c43f37e7 100644 --- a/lib/gitlab/metrics/samplers/ruby_sampler.rb +++ b/lib/gitlab/metrics/samplers/ruby_sampler.rb @@ -23,25 +23,32 @@ module Gitlab end def init_metrics - metrics = {} - metrics[:sampler_duration] = ::Gitlab::Metrics.counter(with_prefix(:sampler, :duration_seconds_total), 'Sampler time', labels) - metrics[:total_time] = ::Gitlab::Metrics.counter(with_prefix(:gc, :duration_seconds_total), 'Total GC time', labels) + metrics = { + file_descriptors: ::Gitlab::Metrics.gauge(with_prefix(:file, :descriptors), 'File descriptors used', labels, :livesum), + memory_bytes: ::Gitlab::Metrics.gauge(with_prefix(:memory, :bytes), 'Memory used', labels, :livesum), + process_cpu_seconds_total: ::Gitlab::Metrics.gauge(with_prefix(:process, :cpu_seconds_total), 'Process CPU seconds total'), + process_max_fds: ::Gitlab::Metrics.gauge(with_prefix(:process, :max_fds), 'Process max fds'), + process_resident_memory_bytes: ::Gitlab::Metrics.gauge(with_prefix(:process, :resident_memory_bytes), 'Memory used', labels, :livesum), + process_start_time_seconds: ::Gitlab::Metrics.gauge(with_prefix(:process, :start_time_seconds), 'Process start time seconds'), + sampler_duration: ::Gitlab::Metrics.counter(with_prefix(:sampler, :duration_seconds_total), 'Sampler time', labels), + total_time: ::Gitlab::Metrics.counter(with_prefix(:gc, :duration_seconds_total), 'Total GC time', labels) + } + GC.stat.keys.each do |key| metrics[key] = ::Gitlab::Metrics.gauge(with_prefix(:gc_stat, key), to_doc_string(key), labels, :livesum) end - metrics[:memory_usage] = ::Gitlab::Metrics.gauge(with_prefix(:memory, :bytes), 'Memory used', labels, :livesum) - metrics[:file_descriptors] = ::Gitlab::Metrics.gauge(with_prefix(:file, :descriptors), 'File descriptors used', labels, :livesum) - metrics end def sample start_time = System.monotonic_time - metrics[:memory_usage].set(labels.merge(worker_label), System.memory_usage) metrics[:file_descriptors].set(labels.merge(worker_label), System.file_descriptor_count) - + metrics[:process_cpu_seconds_total].set(labels.merge(worker_label), ::Gitlab::Metrics::System.cpu_time) + metrics[:process_max_fds].set(labels.merge(worker_label), ::Gitlab::Metrics::System.max_open_file_descriptors) + metrics[:process_start_time_seconds].set(labels.merge(worker_label), ::Gitlab::Metrics::System.process_start_time) + set_memory_usage_metrics sample_gc metrics[:sampler_duration].increment(labels, System.monotonic_time - start_time) @@ -61,6 +68,14 @@ module Gitlab metrics[:total_time].increment(labels, GC::Profiler.total_time) end + def set_memory_usage_metrics + memory_usage = System.memory_usage + memory_labels = labels.merge(worker_label) + + metrics[:memory_bytes].set(memory_labels, memory_usage) + metrics[:process_resident_memory_bytes].set(memory_labels, memory_usage) + end + def worker_label return {} unless defined?(Unicorn::Worker) diff --git a/lib/gitlab/metrics/samplers/unicorn_sampler.rb b/lib/gitlab/metrics/samplers/unicorn_sampler.rb index bec64e864b3..9af7e0afed4 100644 --- a/lib/gitlab/metrics/samplers/unicorn_sampler.rb +++ b/lib/gitlab/metrics/samplers/unicorn_sampler.rb @@ -4,16 +4,16 @@ module Gitlab module Metrics module Samplers class UnicornSampler < BaseSampler - def initialize(interval) - super(interval) + def metrics + @metrics ||= init_metrics end - def unicorn_active_connections - @unicorn_active_connections ||= ::Gitlab::Metrics.gauge(:unicorn_active_connections, 'Unicorn active connections', {}, :max) - end - - def unicorn_queued_connections - @unicorn_queued_connections ||= ::Gitlab::Metrics.gauge(:unicorn_queued_connections, 'Unicorn queued connections', {}, :max) + def init_metrics + { + unicorn_active_connections: ::Gitlab::Metrics.gauge(:unicorn_active_connections, 'Unicorn active connections', {}, :max), + unicorn_queued_connections: ::Gitlab::Metrics.gauge(:unicorn_queued_connections, 'Unicorn queued connections', {}, :max), + unicorn_workers: ::Gitlab::Metrics.gauge(:unicorn_workers, 'Unicorn workers') + } end def enabled? @@ -23,14 +23,13 @@ module Gitlab def sample Raindrops::Linux.tcp_listener_stats(tcp_listeners).each do |addr, stats| - unicorn_active_connections.set({ socket_type: 'tcp', socket_address: addr }, stats.active) - unicorn_queued_connections.set({ socket_type: 'tcp', socket_address: addr }, stats.queued) + set_unicorn_connection_metrics('tcp', addr, stats) end - Raindrops::Linux.unix_listener_stats(unix_listeners).each do |addr, stats| - unicorn_active_connections.set({ socket_type: 'unix', socket_address: addr }, stats.active) - unicorn_queued_connections.set({ socket_type: 'unix', socket_address: addr }, stats.queued) + set_unicorn_connection_metrics('unix', addr, stats) end + + metrics[:unicorn_workers].set({}, unicorn_workers_count) end private @@ -39,6 +38,13 @@ module Gitlab @tcp_listeners ||= Unicorn.listener_names.grep(%r{\A[^/]+:\d+\z}) end + def set_unicorn_connection_metrics(type, addr, stats) + labels = { socket_type: type, socket_address: addr } + + metrics[:unicorn_active_connections].set(labels, stats.active) + metrics[:unicorn_queued_connections].set(labels, stats.queued) + end + def unix_listeners @unix_listeners ||= Unicorn.listener_names - tcp_listeners end @@ -46,6 +52,10 @@ module Gitlab def unicorn_with_listeners? defined?(Unicorn) && Unicorn.listener_names.any? end + + def unicorn_workers_count + `pgrep -f '[u]nicorn_rails worker.+ #{Rails.root.to_s}'`.split.count + end end end end |