From 4c248c05cbd1356199cc96775b68fbbde64d4d5d Mon Sep 17 00:00:00 2001 From: Ryan Cobb Date: Thu, 18 Apr 2019 12:40:00 -0600 Subject: Adds new metrics for unicorn monitoring This adds new metrics for monitoring unicorn. These metrics include process_cpu_seconds_total, process_start_time_seconds, process_max_fds, and unicorn_workers. --- lib/gitlab/metrics/samplers/unicorn_sampler.rb | 36 ++++++++++++++++++++------ lib/gitlab/metrics/system.rb | 25 ++++++++++++++++++ 2 files changed, 53 insertions(+), 8 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/metrics/samplers/unicorn_sampler.rb b/lib/gitlab/metrics/samplers/unicorn_sampler.rb index bec64e864b3..16a2ee9b9be 100644 --- a/lib/gitlab/metrics/samplers/unicorn_sampler.rb +++ b/lib/gitlab/metrics/samplers/unicorn_sampler.rb @@ -8,12 +8,19 @@ module Gitlab super(interval) end - def unicorn_active_connections - @unicorn_active_connections ||= ::Gitlab::Metrics.gauge(:unicorn_active_connections, 'Unicorn active connections', {}, :max) + def metrics + @metrics ||= init_metrics end - def unicorn_queued_connections - @unicorn_queued_connections ||= ::Gitlab::Metrics.gauge(:unicorn_queued_connections, 'Unicorn queued connections', {}, :max) + def init_metrics + { + unicorn_active_connections: ::Gitlab::Metrics.gauge(:unicorn_active_connections, 'Unicorn active connections', {}, :max), + unicorn_queued_connections: ::Gitlab::Metrics.gauge(:unicorn_queued_connections, 'Unicorn queued connections', {}, :max), + unicorn_workers: ::Gitlab::Metrics.gauge(:unicorn_workers, 'Unicorn workers'), + process_cpu_seconds_total: ::Gitlab::Metrics.gauge(:process_cpu_seconds_total, 'Process CPU seconds total'), + process_max_fds: ::Gitlab::Metrics.gauge(:process_max_fds, 'Process max fds'), + process_start_time_seconds: ::Gitlab::Metrics.gauge(:process_start_time_seconds, 'Process start time seconds') + } end def enabled? @@ -23,14 +30,19 @@ module Gitlab def sample Raindrops::Linux.tcp_listener_stats(tcp_listeners).each do |addr, stats| - unicorn_active_connections.set({ socket_type: 'tcp', socket_address: addr }, stats.active) - unicorn_queued_connections.set({ socket_type: 'tcp', socket_address: addr }, stats.queued) + metrics[:unicorn_active_connections].set({ socket_type: 'tcp', socket_address: addr }, stats.active) + metrics[:unicorn_queued_connections].set({ socket_type: 'tcp', socket_address: addr }, stats.queued) end Raindrops::Linux.unix_listener_stats(unix_listeners).each do |addr, stats| - unicorn_active_connections.set({ socket_type: 'unix', socket_address: addr }, stats.active) - unicorn_queued_connections.set({ socket_type: 'unix', socket_address: addr }, stats.queued) + metrics[:unicorn_active_connections].set({ socket_type: 'unix', socket_address: addr }, stats.active) + metrics[:unicorn_queued_connections].set({ socket_type: 'unix', socket_address: addr }, stats.queued) end + + metrics[:process_cpu_seconds_total].set({ pid: nil }, ::Gitlab::Metrics::System.cpu_time) + metrics[:process_start_time_seconds].set({ pid: nil }, ::Gitlab::Metrics::System.process_start_time) + metrics[:process_max_fds].set({ pid: nil }, ::Gitlab::Metrics::System.max_open_file_descriptors) + metrics[:unicorn_workers].set({}, unicorn_workers_count) end private @@ -39,6 +51,10 @@ module Gitlab @tcp_listeners ||= Unicorn.listener_names.grep(%r{\A[^/]+:\d+\z}) end + def pid + @pid ||= Process.pid + end + def unix_listeners @unix_listeners ||= Unicorn.listener_names - tcp_listeners end @@ -46,6 +62,10 @@ module Gitlab def unicorn_with_listeners? defined?(Unicorn) && Unicorn.listener_names.any? end + + def unicorn_workers_count + Sys::ProcTable.ps.select {|p| p.cmdline.match(/unicorn_rails worker/)}.count + end end end end diff --git a/lib/gitlab/metrics/system.rb b/lib/gitlab/metrics/system.rb index 426496855e3..a269a8688e9 100644 --- a/lib/gitlab/metrics/system.rb +++ b/lib/gitlab/metrics/system.rb @@ -23,6 +23,16 @@ module Gitlab def self.file_descriptor_count Dir.glob('/proc/self/fd/*').length end + + def self.max_open_file_descriptors + match = File.read('/proc/self/limits').match(/Max open files\s*(\d+)/) + + if match && match[1] + max_fds = match[1].to_i + end + + max_fds + end else def self.memory_usage 0.0 @@ -31,6 +41,10 @@ module Gitlab def self.file_descriptor_count 0 end + + def self.max_open_file_descriptors + 0 + end end # THREAD_CPUTIME is not supported on OS X @@ -46,6 +60,17 @@ module Gitlab end end + # CLOCK_BOOTTIME is not supported on OS X + if Process.const_defined?(:CLOCK_BOOTTIME) + def self.process_start_time + Process + .clock_gettime(Process::CLOCK_BOOTTIME, :float_second) + end + else + def self.process_start_time + 0.0 + end + end # Returns the current real time in a given precision. # # Returns the time as a Float for precision = :float_second. -- cgit v1.2.3 From 174a03dfc284781d811df1874ce3cf11d451a8f5 Mon Sep 17 00:00:00 2001 From: Ryan Cobb Date: Wed, 24 Apr 2019 11:41:54 -0600 Subject: Move process specific metrics to ruby sampler These metrics are not unicorn specific and can be used across ruby processes --- lib/gitlab/metrics/samplers/ruby_sampler.rb | 22 ++++++++++++++-------- lib/gitlab/metrics/samplers/unicorn_sampler.rb | 8 +------- 2 files changed, 15 insertions(+), 15 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/metrics/samplers/ruby_sampler.rb b/lib/gitlab/metrics/samplers/ruby_sampler.rb index 18a69321905..c4b2224efdf 100644 --- a/lib/gitlab/metrics/samplers/ruby_sampler.rb +++ b/lib/gitlab/metrics/samplers/ruby_sampler.rb @@ -23,25 +23,31 @@ module Gitlab end def init_metrics - metrics = {} - metrics[:sampler_duration] = ::Gitlab::Metrics.counter(with_prefix(:sampler, :duration_seconds_total), 'Sampler time', labels) - metrics[:total_time] = ::Gitlab::Metrics.counter(with_prefix(:gc, :duration_seconds_total), 'Total GC time', labels) + metrics = { + file_descriptors: ::Gitlab::Metrics.gauge(with_prefix(:file, :descriptors), 'File descriptors used', labels, :livesum), + memory_usage: ::Gitlab::Metrics.gauge(with_prefix(:memory, :bytes), 'Memory used', labels, :livesum), + process_cpu_seconds_total: ::Gitlab::Metrics.gauge(:process_cpu_seconds_total, 'Process CPU seconds total'), + process_max_fds: ::Gitlab::Metrics.gauge(:process_max_fds, 'Process max fds'), + process_start_time_seconds: ::Gitlab::Metrics.gauge(:process_start_time_seconds, 'Process start time seconds'), + sampler_duration: ::Gitlab::Metrics.counter(with_prefix(:sampler, :duration_seconds_total), 'Sampler time', labels), + total_time: ::Gitlab::Metrics.counter(with_prefix(:gc, :duration_seconds_total), 'Total GC time', labels) + } + GC.stat.keys.each do |key| metrics[key] = ::Gitlab::Metrics.gauge(with_prefix(:gc_stat, key), to_doc_string(key), labels, :livesum) end - metrics[:memory_usage] = ::Gitlab::Metrics.gauge(with_prefix(:memory, :bytes), 'Memory used', labels, :livesum) - metrics[:file_descriptors] = ::Gitlab::Metrics.gauge(with_prefix(:file, :descriptors), 'File descriptors used', labels, :livesum) - metrics end def sample start_time = System.monotonic_time - metrics[:memory_usage].set(labels.merge(worker_label), System.memory_usage) metrics[:file_descriptors].set(labels.merge(worker_label), System.file_descriptor_count) - + metrics[:memory_usage].set(labels.merge(worker_label), System.memory_usage) + metrics[:process_cpu_seconds_total].set(labels.merge(worker_label), ::Gitlab::Metrics::System.cpu_time) + metrics[:process_start_time_seconds].set(labels.merge(worker_label), ::Gitlab::Metrics::System.process_start_time) + metrics[:process_max_fds].set(labels.merge(worker_label), ::Gitlab::Metrics::System.max_open_file_descriptors) sample_gc metrics[:sampler_duration].increment(labels, System.monotonic_time - start_time) diff --git a/lib/gitlab/metrics/samplers/unicorn_sampler.rb b/lib/gitlab/metrics/samplers/unicorn_sampler.rb index 16a2ee9b9be..c7063c5ba28 100644 --- a/lib/gitlab/metrics/samplers/unicorn_sampler.rb +++ b/lib/gitlab/metrics/samplers/unicorn_sampler.rb @@ -16,10 +16,7 @@ module Gitlab { unicorn_active_connections: ::Gitlab::Metrics.gauge(:unicorn_active_connections, 'Unicorn active connections', {}, :max), unicorn_queued_connections: ::Gitlab::Metrics.gauge(:unicorn_queued_connections, 'Unicorn queued connections', {}, :max), - unicorn_workers: ::Gitlab::Metrics.gauge(:unicorn_workers, 'Unicorn workers'), - process_cpu_seconds_total: ::Gitlab::Metrics.gauge(:process_cpu_seconds_total, 'Process CPU seconds total'), - process_max_fds: ::Gitlab::Metrics.gauge(:process_max_fds, 'Process max fds'), - process_start_time_seconds: ::Gitlab::Metrics.gauge(:process_start_time_seconds, 'Process start time seconds') + unicorn_workers: ::Gitlab::Metrics.gauge(:unicorn_workers, 'Unicorn workers') } end @@ -39,9 +36,6 @@ module Gitlab metrics[:unicorn_queued_connections].set({ socket_type: 'unix', socket_address: addr }, stats.queued) end - metrics[:process_cpu_seconds_total].set({ pid: nil }, ::Gitlab::Metrics::System.cpu_time) - metrics[:process_start_time_seconds].set({ pid: nil }, ::Gitlab::Metrics::System.process_start_time) - metrics[:process_max_fds].set({ pid: nil }, ::Gitlab::Metrics::System.max_open_file_descriptors) metrics[:unicorn_workers].set({}, unicorn_workers_count) end -- cgit v1.2.3 From bb27bf4a1c7153f2f5074eb058d8659dd9f198ad Mon Sep 17 00:00:00 2001 From: Ryan Cobb Date: Wed, 24 Apr 2019 17:05:09 -0600 Subject: Update docs and calculate process start time via proc table This updates monitor docs to reflect the new ruby and unicorn metrics as well as making it so we fetch process start time via the proc table instead of via CLOCK_BOOTTIME --- lib/gitlab/metrics/samplers/ruby_sampler.rb | 18 +++++++++--------- lib/gitlab/metrics/system.rb | 22 +++++++++++----------- 2 files changed, 20 insertions(+), 20 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/metrics/samplers/ruby_sampler.rb b/lib/gitlab/metrics/samplers/ruby_sampler.rb index c4b2224efdf..5740380e63e 100644 --- a/lib/gitlab/metrics/samplers/ruby_sampler.rb +++ b/lib/gitlab/metrics/samplers/ruby_sampler.rb @@ -24,13 +24,13 @@ module Gitlab def init_metrics metrics = { - file_descriptors: ::Gitlab::Metrics.gauge(with_prefix(:file, :descriptors), 'File descriptors used', labels, :livesum), - memory_usage: ::Gitlab::Metrics.gauge(with_prefix(:memory, :bytes), 'Memory used', labels, :livesum), - process_cpu_seconds_total: ::Gitlab::Metrics.gauge(:process_cpu_seconds_total, 'Process CPU seconds total'), - process_max_fds: ::Gitlab::Metrics.gauge(:process_max_fds, 'Process max fds'), - process_start_time_seconds: ::Gitlab::Metrics.gauge(:process_start_time_seconds, 'Process start time seconds'), - sampler_duration: ::Gitlab::Metrics.counter(with_prefix(:sampler, :duration_seconds_total), 'Sampler time', labels), - total_time: ::Gitlab::Metrics.counter(with_prefix(:gc, :duration_seconds_total), 'Total GC time', labels) + file_descriptors: ::Gitlab::Metrics.gauge(with_prefix(:file, :descriptors), 'File descriptors used', labels, :livesum), + process_cpu_seconds_total: ::Gitlab::Metrics.gauge(with_prefix(:process, :cpu_seconds_total), 'Process CPU seconds total'), + process_max_fds: ::Gitlab::Metrics.gauge(with_prefix(:process, :max_fds), 'Process max fds'), + process_resident_memory_bytes: ::Gitlab::Metrics.gauge(with_prefix(:process, :resident_memory_bytes), 'Memory used', labels, :livesum), + process_start_time_seconds: ::Gitlab::Metrics.gauge(with_prefix(:process, :start_time_seconds), 'Process start time seconds'), + sampler_duration: ::Gitlab::Metrics.counter(with_prefix(:sampler, :duration_seconds_total), 'Sampler time', labels), + total_time: ::Gitlab::Metrics.counter(with_prefix(:gc, :duration_seconds_total), 'Total GC time', labels) } GC.stat.keys.each do |key| @@ -44,10 +44,10 @@ module Gitlab start_time = System.monotonic_time metrics[:file_descriptors].set(labels.merge(worker_label), System.file_descriptor_count) - metrics[:memory_usage].set(labels.merge(worker_label), System.memory_usage) metrics[:process_cpu_seconds_total].set(labels.merge(worker_label), ::Gitlab::Metrics::System.cpu_time) - metrics[:process_start_time_seconds].set(labels.merge(worker_label), ::Gitlab::Metrics::System.process_start_time) metrics[:process_max_fds].set(labels.merge(worker_label), ::Gitlab::Metrics::System.max_open_file_descriptors) + metrics[:process_resident_memory_bytes].set(labels.merge(worker_label), System.memory_usage) + metrics[:process_start_time_seconds].set(labels.merge(worker_label), ::Gitlab::Metrics::System.process_start_time) sample_gc metrics[:sampler_duration].increment(labels, System.monotonic_time - start_time) diff --git a/lib/gitlab/metrics/system.rb b/lib/gitlab/metrics/system.rb index a269a8688e9..ecd558d7ec7 100644 --- a/lib/gitlab/metrics/system.rb +++ b/lib/gitlab/metrics/system.rb @@ -33,6 +33,13 @@ module Gitlab max_fds end + + def self.process_start_time + start_time_in_jiffies = Sys::ProcTable.ps(pid: Process.pid).starttime + return 0 unless start_time_in_jiffies + + start_time_in_jiffies / 100 + end else def self.memory_usage 0.0 @@ -45,6 +52,10 @@ module Gitlab def self.max_open_file_descriptors 0 end + + def self.process_start_time + 0 + end end # THREAD_CPUTIME is not supported on OS X @@ -60,17 +71,6 @@ module Gitlab end end - # CLOCK_BOOTTIME is not supported on OS X - if Process.const_defined?(:CLOCK_BOOTTIME) - def self.process_start_time - Process - .clock_gettime(Process::CLOCK_BOOTTIME, :float_second) - end - else - def self.process_start_time - 0.0 - end - end # Returns the current real time in a given precision. # # Returns the time as a Float for precision = :float_second. -- cgit v1.2.3 From 2f6a1e77bc8d3b57eddc337ec8f4485d0f26b32c Mon Sep 17 00:00:00 2001 From: Ryan Cobb Date: Fri, 26 Apr 2019 10:16:53 -0600 Subject: Cleanup syntax in System max_open_file_descriptors --- lib/gitlab/metrics/system.rb | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/metrics/system.rb b/lib/gitlab/metrics/system.rb index ecd558d7ec7..ce0bb82d138 100644 --- a/lib/gitlab/metrics/system.rb +++ b/lib/gitlab/metrics/system.rb @@ -27,11 +27,9 @@ module Gitlab def self.max_open_file_descriptors match = File.read('/proc/self/limits').match(/Max open files\s*(\d+)/) - if match && match[1] - max_fds = match[1].to_i - end + return unless match && match[1] - max_fds + match[1].to_i end def self.process_start_time -- cgit v1.2.3 From 17986d91a5a54a8f68c45818073e77689f3c1fd6 Mon Sep 17 00:00:00 2001 From: Ryan Cobb Date: Mon, 29 Apr 2019 12:13:02 -0600 Subject: Add back ruby_memory_bytes metric, limit duplication, clean up This adds back ruby_memory_bytes for backwards compatibility, limits code duplication, cleans up unused methods, and limits the unicorn worker sampling scope. --- lib/gitlab/metrics/samplers/ruby_sampler.rb | 11 ++++++++++- lib/gitlab/metrics/samplers/unicorn_sampler.rb | 16 ++++++++-------- 2 files changed, 18 insertions(+), 9 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/metrics/samplers/ruby_sampler.rb b/lib/gitlab/metrics/samplers/ruby_sampler.rb index 5740380e63e..4d9c43f37e7 100644 --- a/lib/gitlab/metrics/samplers/ruby_sampler.rb +++ b/lib/gitlab/metrics/samplers/ruby_sampler.rb @@ -25,6 +25,7 @@ module Gitlab def init_metrics metrics = { file_descriptors: ::Gitlab::Metrics.gauge(with_prefix(:file, :descriptors), 'File descriptors used', labels, :livesum), + memory_bytes: ::Gitlab::Metrics.gauge(with_prefix(:memory, :bytes), 'Memory used', labels, :livesum), process_cpu_seconds_total: ::Gitlab::Metrics.gauge(with_prefix(:process, :cpu_seconds_total), 'Process CPU seconds total'), process_max_fds: ::Gitlab::Metrics.gauge(with_prefix(:process, :max_fds), 'Process max fds'), process_resident_memory_bytes: ::Gitlab::Metrics.gauge(with_prefix(:process, :resident_memory_bytes), 'Memory used', labels, :livesum), @@ -46,8 +47,8 @@ module Gitlab metrics[:file_descriptors].set(labels.merge(worker_label), System.file_descriptor_count) metrics[:process_cpu_seconds_total].set(labels.merge(worker_label), ::Gitlab::Metrics::System.cpu_time) metrics[:process_max_fds].set(labels.merge(worker_label), ::Gitlab::Metrics::System.max_open_file_descriptors) - metrics[:process_resident_memory_bytes].set(labels.merge(worker_label), System.memory_usage) metrics[:process_start_time_seconds].set(labels.merge(worker_label), ::Gitlab::Metrics::System.process_start_time) + set_memory_usage_metrics sample_gc metrics[:sampler_duration].increment(labels, System.monotonic_time - start_time) @@ -67,6 +68,14 @@ module Gitlab metrics[:total_time].increment(labels, GC::Profiler.total_time) end + def set_memory_usage_metrics + memory_usage = System.memory_usage + memory_labels = labels.merge(worker_label) + + metrics[:memory_bytes].set(memory_labels, memory_usage) + metrics[:process_resident_memory_bytes].set(memory_labels, memory_usage) + end + def worker_label return {} unless defined?(Unicorn::Worker) diff --git a/lib/gitlab/metrics/samplers/unicorn_sampler.rb b/lib/gitlab/metrics/samplers/unicorn_sampler.rb index c7063c5ba28..1b6c52ac0bf 100644 --- a/lib/gitlab/metrics/samplers/unicorn_sampler.rb +++ b/lib/gitlab/metrics/samplers/unicorn_sampler.rb @@ -27,13 +27,10 @@ module Gitlab def sample Raindrops::Linux.tcp_listener_stats(tcp_listeners).each do |addr, stats| - metrics[:unicorn_active_connections].set({ socket_type: 'tcp', socket_address: addr }, stats.active) - metrics[:unicorn_queued_connections].set({ socket_type: 'tcp', socket_address: addr }, stats.queued) + set_unicorn_connection_metrics('tcp', addr, stats) end - Raindrops::Linux.unix_listener_stats(unix_listeners).each do |addr, stats| - metrics[:unicorn_active_connections].set({ socket_type: 'unix', socket_address: addr }, stats.active) - metrics[:unicorn_queued_connections].set({ socket_type: 'unix', socket_address: addr }, stats.queued) + set_unicorn_connection_metrics('unix', addr, stats) end metrics[:unicorn_workers].set({}, unicorn_workers_count) @@ -45,8 +42,11 @@ module Gitlab @tcp_listeners ||= Unicorn.listener_names.grep(%r{\A[^/]+:\d+\z}) end - def pid - @pid ||= Process.pid + def set_unicorn_connection_metrics(type, addr, stats) + labels = { socket_type: type, socket_address: addr } + + metrics[:unicorn_active_connections].set(labels, stats.active) + metrics[:unicorn_queued_connections].set(labels, stats.queued) end def unix_listeners @@ -58,7 +58,7 @@ module Gitlab end def unicorn_workers_count - Sys::ProcTable.ps.select {|p| p.cmdline.match(/unicorn_rails worker/)}.count + Sys::ProcTable.ps.select {|p| p.cmdline.match(/unicorn_rails worker.+ #{Rails.root.to_s}/)}.count end end end -- cgit v1.2.3