Transaction and method instrumentation

3 files changed, 58 insertions, 26 deletions

diff --git a/lib/gitlab/metrics/method_call.rb b/lib/gitlab/metrics/method_call.rb
index d3465e5ec19..2c650a987f1 100644
--- a/lib/gitlab/metrics/method_call.rb
+++ b/lib/gitlab/metrics/method_call.rb
@@ -7,13 +7,28 @@ module Gitlab
       # name - The full name of the method (including namespace) such as
       #        `User#sign_in`.
       #
-      # series - The series to use for storing the data.
-      def initialize(name, series)
+      def self.call_real_duration_histogram
+        @call_real_duration_histogram ||= Gitlab::Metrics.histogram(:gitlab_method_call_real_duration_milliseconds,
+                                                                    'Method calls real duration',
+                                                                    {},
+                                                                    [1, 2, 5, 10, 20, 50, 100, 1000])
+
+      end
+
+      def self.call_cpu_duration_histogram
+        @call_duration_histogram ||= Gitlab::Metrics.histogram(:gitlab_method_call_cpu_duration_milliseconds,
+                                                               'Method calls cpu duration',
+                                                               {},
+                                                               [1, 2, 5, 10, 20, 50, 100, 1000])
+      end
+
+
+      def initialize(name, tags = {})
         @name = name
-        @series = series
         @real_time = 0
         @cpu_time = 0
         @call_count = 0
+        @tags = tags
       end
 
       # Measures the real and CPU execution time of the supplied block.
@@ -26,17 +41,34 @@ module Gitlab
         @cpu_time += System.cpu_time - start_cpu
         @call_count += 1
 
+        if above_threshold?
+          self.class.call_real_duration_histogram.observe(labels, @real_time)
+          self.class.call_cpu_duration_histogram.observe(labels, @cpu_time)
+        end
+
         retval
       end
 
+      def labels
+        @labels ||= @tags.merge(source_label).merge({ call_name: @name })
+      end
+
+      def source_label
+        if Sidekiq.server?
+          { source: 'sidekiq' }
+        else
+          { source: 'rails' }
+        end
+      end
+
       # Returns a Metric instance of the current method call.
       def to_metric
         Metric.new(
-          @series,
+          Instrumentation.series,
           {
-            duration:     real_time,
+            duration: real_time,
             cpu_duration: cpu_time,
-            call_count:   call_count
+            call_count: call_count
           },
           method: @name
         )
diff --git a/lib/gitlab/metrics/samplers/ruby_sampler.rb b/lib/gitlab/metrics/samplers/ruby_sampler.rb
index e61356bb460..61eb6e7b541 100644
--- a/lib/gitlab/metrics/samplers/ruby_sampler.rb
+++ b/lib/gitlab/metrics/samplers/ruby_sampler.rb
@@ -17,7 +17,7 @@ module Gitlab
         end
 
         def labels
-          worker_label.merge(source_label)
+          {}
         end
 
         def initialize(interval)
@@ -53,7 +53,7 @@ module Gitlab
           metrics[:memory_usage].set(labels, System.memory_usage)
           metrics[:file_descriptors].set(labels, System.file_descriptor_count)
 
-          metrics[:sampler_duration].observe(source_label, (System.monotonic_time - start_time) / 1000.0)
+          metrics[:sampler_duration].observe(labels.merge(worker_label), (System.monotonic_time - start_time) / 1000.0)
         ensure
           GC::Profiler.clear
         end
@@ -94,14 +94,6 @@ module Gitlab
           end
         end
 
-        def source_label
-          if Sidekiq.server?
-            { source: 'sidekiq' }
-          else
-            { source: 'rails' }
-          end
-        end
-
         def worker_label
           return {} unless defined?(Unicorn::Worker)
           worker_no = ::Prometheus::Client::Support::Unicorn.worker_id
diff --git a/lib/gitlab/metrics/transaction.rb b/lib/gitlab/metrics/transaction.rb
index 4f9fb1c7853..2a4578aa6e4 100644
--- a/lib/gitlab/metrics/transaction.rb
+++ b/lib/gitlab/metrics/transaction.rb
@@ -21,15 +21,15 @@ module Gitlab
         @metrics = []
         @methods = {}
 
-        @started_at  = nil
+        @started_at = nil
         @finished_at = nil
 
         @values = Hash.new(0)
-        @tags   = {}
+        @tags = {}
         @action = action
 
         @memory_before = 0
-        @memory_after  = 0
+        @memory_after = 0
       end
 
       def duration
@@ -44,12 +44,17 @@ module Gitlab
         Thread.current[THREAD_KEY] = self
 
         @memory_before = System.memory_usage
-        @started_at    = System.monotonic_time
+        @started_at = System.monotonic_time
 
         yield
       ensure
         @memory_after = System.memory_usage
-        @finished_at  = System.monotonic_time
+        @finished_at = System.monotonic_time
+
+        Gitlab::Metrics.histogram("gitlab_method_duration_seconds".to_sym, "Method duration seconds", @tags).observe({}, )
+
+        self.class.prometheus_gauge(:duration).set(@tags, duration)
+        self.class.prometheus_gauge(:allocated_memory).set(@tags, allocated_memory)
 
         Thread.current[THREAD_KEY] = nil
       end
@@ -66,16 +71,14 @@ module Gitlab
       # event_name - The name of the event (e.g. "git_push").
       # tags - A set of tags to attach to the event.
       def add_event(event_name, tags = {})
-        @metrics << Metric.new(EVENT_SERIES,
-                               { count: 1 },
-                               { event: event_name }.merge(tags),
-                               :event)
+        Gitlab::Metrics.counter("gitlab_event_#{event_name}".to_sym, "Event #{event_name}", tags).increment({})
+        @metrics << Metric.new(EVENT_SERIES, { count: 1 }, labels, :event)
       end
 
       # Returns a MethodCall object for the given name.
       def method_call_for(name)
         unless method = @methods[name]
-          @methods[name] = method = MethodCall.new(name, Instrumentation.series)
+          @methods[name] = method = MethodCall.new(name)
         end
 
         method
@@ -103,11 +106,16 @@ module Gitlab
 
         @values.each do |name, value|
           values[name] = value
+          self.class.prometheus_gauge(name).set(@tags, value)
         end
 
         add_metric('transactions', values, @tags)
       end
 
+      def self.prometheus_gauge(name)
+        Gitlab::Metrics.gauge("gitlab_transaction_#{name}".to_sym, "Gitlab Transaction #{name}")
+      end
+
       def submit
         submit = @metrics.dup