From 1f9edb7c4a393a6ebe78e6f98e46515ad655cece Mon Sep 17 00:00:00 2001
From: Aleksei Lipniagov <alipniagov@gitlab.com>
Date: Fri, 2 Aug 2019 09:04:32 +0000
Subject: Call `GC::Profiler.clear` only in one place

Previously, both InfluxSampler and RubySampler were relying on the
`GC::Profiler.total_time` data which is the sum over the list
of captured GC events. Also, both samplers asynchronously called
`GC::Profiler.clear` which led to incorrect metric data because
each sampler has the wrong assumption it is the only object who calls
`GC::Profiler.clear` and thus could rely on the gathered results between
such calls.

We should ensure that `GC::Profiler.total_time` is called only in one
place making it possible to rely on accumulated data between such wipes.

Also, we need to track the amount of profiler reports we lost.
---
 lib/gitlab/metrics/samplers/influx_sampler.rb | 22 ----------------------
 lib/gitlab/metrics/samplers/ruby_sampler.rb   | 23 +++++++++++++++++------
 2 files changed, 17 insertions(+), 28 deletions(-)

(limited to 'lib/gitlab')

diff --git a/lib/gitlab/metrics/samplers/influx_sampler.rb b/lib/gitlab/metrics/samplers/influx_sampler.rb
index 5138b37f83e..1eae0a7bf45 100644
--- a/lib/gitlab/metrics/samplers/influx_sampler.rb
+++ b/lib/gitlab/metrics/samplers/influx_sampler.rb
@@ -15,19 +15,14 @@ module Gitlab
           @last_step = nil
 
           @metrics = []
-
-          @last_minor_gc = Delta.new(GC.stat[:minor_gc_count])
-          @last_major_gc = Delta.new(GC.stat[:major_gc_count])
         end
 
         def sample
           sample_memory_usage
           sample_file_descriptors
-          sample_gc
 
           flush
         ensure
-          GC::Profiler.clear
           @metrics.clear
         end
 
@@ -43,23 +38,6 @@ module Gitlab
           add_metric('file_descriptors', value: System.file_descriptor_count)
         end
 
-        def sample_gc
-          time = GC::Profiler.total_time * 1000.0
-          stats = GC.stat.merge(total_time: time)
-
-          # We want the difference of GC runs compared to the last sample, not the
-          # total amount since the process started.
-          stats[:minor_gc_count] =
-            @last_minor_gc.compared_with(stats[:minor_gc_count])
-
-          stats[:major_gc_count] =
-            @last_major_gc.compared_with(stats[:major_gc_count])
-
-          stats[:count] = stats[:minor_gc_count] + stats[:major_gc_count]
-
-          add_metric('gc_statistics', stats)
-        end
-
         def add_metric(series, values, tags = {})
           prefix = sidekiq? ? 'sidekiq_' : 'rails_'
 
diff --git a/lib/gitlab/metrics/samplers/ruby_sampler.rb b/lib/gitlab/metrics/samplers/ruby_sampler.rb
index 1e200db0baf..3bfa3da35e0 100644
--- a/lib/gitlab/metrics/samplers/ruby_sampler.rb
+++ b/lib/gitlab/metrics/samplers/ruby_sampler.rb
@@ -6,7 +6,11 @@ module Gitlab
   module Metrics
     module Samplers
       class RubySampler < BaseSampler
+        GC_REPORT_BUCKETS = [0.001, 0.002, 0.005, 0.01, 0.05, 0.1, 0.5].freeze
+
         def initialize(interval)
+          GC::Profiler.clear
+
           metrics[:process_start_time_seconds].set(labels, Time.now.to_i)
 
           super
@@ -37,7 +41,7 @@ module Gitlab
             process_resident_memory_bytes:  ::Gitlab::Metrics.gauge(with_prefix(:process, :resident_memory_bytes), 'Memory used', labels),
             process_start_time_seconds:     ::Gitlab::Metrics.gauge(with_prefix(:process, :start_time_seconds), 'Process start time seconds'),
             sampler_duration:               ::Gitlab::Metrics.counter(with_prefix(:sampler, :duration_seconds_total), 'Sampler time', labels),
-            total_time:                     ::Gitlab::Metrics.counter(with_prefix(:gc, :duration_seconds_total), 'Total GC time', labels)
+            gc_duration_seconds:            ::Gitlab::Metrics.histogram(with_prefix(:gc, :duration_seconds), 'GC time', labels, GC_REPORT_BUCKETS)
           }
 
           GC.stat.keys.each do |key|
@@ -57,20 +61,27 @@ module Gitlab
           sample_gc
 
           metrics[:sampler_duration].increment(labels, System.monotonic_time - start_time)
-        ensure
-          GC::Profiler.clear
         end
 
         private
 
         def sample_gc
-          # Collect generic GC stats.
+          # Observe all GC samples
+          sample_gc_reports.each do |report|
+            metrics[:gc_duration_seconds].observe(labels, report[:GC_TIME])
+          end
+
+          # Collect generic GC stats
           GC.stat.each do |key, value|
             metrics[key].set(labels, value)
           end
+        end
 
-          # Collect the GC time since last sample in float seconds.
-          metrics[:total_time].increment(labels, GC::Profiler.total_time)
+        def sample_gc_reports
+          GC::Profiler.enable
+          GC::Profiler.raw_data
+        ensure
+          GC::Profiler.clear
         end
 
         def set_memory_usage_metrics
-- 
cgit v1.2.3