Add latest changes from gitlab-org/gitlab@13-1-stable-ee

5 files changed, 323 insertions, 51 deletions

diff --git a/lib/gitlab/instrumentation/elasticsearch_transport.rb b/lib/gitlab/instrumentation/elasticsearch_transport.rb
new file mode 100644
index 00000000000..deee0127c0c
--- /dev/null
+++ b/lib/gitlab/instrumentation/elasticsearch_transport.rb
@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+require 'elasticsearch-transport'
+
+module Gitlab
+  module Instrumentation
+    module ElasticsearchTransportInterceptor
+      def perform_request(*args)
+        start = Time.now
+        super
+      ensure
+        if ::Gitlab::SafeRequestStore.active?
+          duration = (Time.now - start)
+
+          ::Gitlab::Instrumentation::ElasticsearchTransport.increment_request_count
+          ::Gitlab::Instrumentation::ElasticsearchTransport.add_duration(duration)
+          ::Gitlab::Instrumentation::ElasticsearchTransport.add_call_details(duration, args)
+        end
+      end
+    end
+
+    class ElasticsearchTransport
+      ELASTICSEARCH_REQUEST_COUNT = :elasticsearch_request_count
+      ELASTICSEARCH_CALL_DURATION = :elasticsearch_call_duration
+      ELASTICSEARCH_CALL_DETAILS = :elasticsearch_call_details
+
+      def self.get_request_count
+        ::Gitlab::SafeRequestStore[ELASTICSEARCH_REQUEST_COUNT] || 0
+      end
+
+      def self.increment_request_count
+        ::Gitlab::SafeRequestStore[ELASTICSEARCH_REQUEST_COUNT] ||= 0
+        ::Gitlab::SafeRequestStore[ELASTICSEARCH_REQUEST_COUNT] += 1
+      end
+
+      def self.detail_store
+        ::Gitlab::SafeRequestStore[ELASTICSEARCH_CALL_DETAILS] ||= []
+      end
+
+      def self.query_time
+        query_time = ::Gitlab::SafeRequestStore[ELASTICSEARCH_CALL_DURATION] || 0
+        query_time.round(::Gitlab::InstrumentationHelper::DURATION_PRECISION)
+      end
+
+      def self.add_duration(duration)
+        ::Gitlab::SafeRequestStore[ELASTICSEARCH_CALL_DURATION] ||= 0
+        ::Gitlab::SafeRequestStore[ELASTICSEARCH_CALL_DURATION] += duration
+      end
+
+      def self.add_call_details(duration, args)
+        return unless Gitlab::PerformanceBar.enabled_for_request?
+
+        detail_store << {
+          method: args[0],
+          path: args[1],
+          params: args[2],
+          body: args[3],
+          duration: duration,
+          backtrace: ::Gitlab::BacktraceCleaner.clean_backtrace(caller)
+        }
+      end
+    end
+  end
+end
+
+class ::Elasticsearch::Transport::Client
+  prepend ::Gitlab::Instrumentation::ElasticsearchTransportInterceptor
+end
diff --git a/lib/gitlab/instrumentation/redis.rb b/lib/gitlab/instrumentation/redis.rb
index cc99e828251..82b4701872f 100644
--- a/lib/gitlab/instrumentation/redis.rb
+++ b/lib/gitlab/instrumentation/redis.rb
@@ -1,67 +1,46 @@
 # frozen_string_literal: true
 
-require 'redis'
-
 module Gitlab
   module Instrumentation
-    module RedisInterceptor
-      def call(*args, &block)
-        start = Time.now
-        super(*args, &block)
-      ensure
-        duration = (Time.now - start)
-
-        if ::RequestStore.active?
-          ::Gitlab::Instrumentation::Redis.increment_request_count
-          ::Gitlab::Instrumentation::Redis.add_duration(duration)
-          ::Gitlab::Instrumentation::Redis.add_call_details(duration, args)
-        end
-      end
-    end
-
+    # Aggregates Redis measurements from different request storage sources.
     class Redis
-      REDIS_REQUEST_COUNT = :redis_request_count
-      REDIS_CALL_DURATION = :redis_call_duration
-      REDIS_CALL_DETAILS = :redis_call_details
+      ActionCable = Class.new(RedisBase)
+      Cache = Class.new(RedisBase)
+      Queues = Class.new(RedisBase)
+      SharedState = Class.new(RedisBase)
 
-      def self.get_request_count
-        ::RequestStore[REDIS_REQUEST_COUNT] || 0
-      end
+      STORAGES = [ActionCable, Cache, Queues, SharedState].freeze
 
-      def self.increment_request_count
-        ::RequestStore[REDIS_REQUEST_COUNT] ||= 0
-        ::RequestStore[REDIS_REQUEST_COUNT] += 1
-      end
+      # Milliseconds represented in seconds (from 1 millisecond to 2 seconds).
+      QUERY_TIME_BUCKETS = [0.001, 0.0025, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2].freeze
 
-      def self.detail_store
-        ::RequestStore[REDIS_CALL_DETAILS] ||= []
-      end
+      class << self
+        include ::Gitlab::Instrumentation::RedisPayload
 
-      def self.query_time
-        query_time = ::RequestStore[REDIS_CALL_DURATION] || 0
-        query_time.round(::Gitlab::InstrumentationHelper::DURATION_PRECISION)
-      end
+        def storage_key
+          nil
+        end
 
-      def self.add_duration(duration)
-        total_time = query_time + duration
-        ::RequestStore[REDIS_CALL_DURATION] = total_time
-      end
+        def known_payload_keys
+          super + STORAGES.flat_map(&:known_payload_keys)
+        end
 
-      def self.add_call_details(duration, args)
-        return unless Gitlab::PerformanceBar.enabled_for_request?
-        # redis-rb passes an array (e.g. [:get, key])
-        return unless args.length == 1
+        def payload
+          super.merge(*STORAGES.flat_map(&:payload))
+        end
 
-        detail_store << {
-          cmd: args.first,
-          duration: duration,
-          backtrace: ::Gitlab::BacktraceCleaner.clean_backtrace(caller)
-        }
+        def detail_store
+          STORAGES.flat_map do |storage|
+            storage.detail_store.map { |details| details.merge(storage: storage.name.demodulize) }
+          end
+        end
+
+        %i[get_request_count query_time read_bytes write_bytes].each do |method|
+          define_method method do
+            STORAGES.sum(&method) # rubocop:disable CodeReuse/ActiveRecord
+          end
+        end
       end
     end
   end
 end
-
-class ::Redis::Client
-  prepend ::Gitlab::Instrumentation::RedisInterceptor
-end
diff --git a/lib/gitlab/instrumentation/redis_base.rb b/lib/gitlab/instrumentation/redis_base.rb
new file mode 100644
index 00000000000..012543e1645
--- /dev/null
+++ b/lib/gitlab/instrumentation/redis_base.rb
@@ -0,0 +1,102 @@
+# frozen_string_literal: true
+
+require 'redis'
+
+module Gitlab
+  module Instrumentation
+    class RedisBase
+      class << self
+        include ::Gitlab::Utils::StrongMemoize
+        include ::Gitlab::Instrumentation::RedisPayload
+
+        # TODO: To be used by https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/395
+        # as a 'label' alias.
+        def storage_key
+          self.name.demodulize.underscore
+        end
+
+        def add_duration(duration)
+          ::RequestStore[call_duration_key] ||= 0
+          ::RequestStore[call_duration_key] += duration
+        end
+
+        def add_call_details(duration, args)
+          return unless Gitlab::PerformanceBar.enabled_for_request?
+          # redis-rb passes an array (e.g. [[:get, key]])
+          return unless args.length == 1
+
+          # TODO: Add information about current Redis client
+          # being instrumented.
+          # https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/316.
+          detail_store << {
+            cmd: args.first,
+            duration: duration,
+            backtrace: ::Gitlab::BacktraceCleaner.clean_backtrace(caller)
+          }
+        end
+
+        def increment_request_count
+          ::RequestStore[request_count_key] ||= 0
+          ::RequestStore[request_count_key] += 1
+        end
+
+        def increment_read_bytes(num_bytes)
+          ::RequestStore[read_bytes_key] ||= 0
+          ::RequestStore[read_bytes_key] += num_bytes
+        end
+
+        def increment_write_bytes(num_bytes)
+          ::RequestStore[write_bytes_key] ||= 0
+          ::RequestStore[write_bytes_key] += num_bytes
+        end
+
+        def get_request_count
+          ::RequestStore[request_count_key] || 0
+        end
+
+        def read_bytes
+          ::RequestStore[read_bytes_key] || 0
+        end
+
+        def write_bytes
+          ::RequestStore[write_bytes_key] || 0
+        end
+
+        def detail_store
+          ::RequestStore[call_details_key] ||= []
+        end
+
+        def query_time
+          query_time = ::RequestStore[call_duration_key] || 0
+          query_time.round(::Gitlab::InstrumentationHelper::DURATION_PRECISION)
+        end
+
+        private
+
+        def request_count_key
+          strong_memoize(:request_count_key) { build_key(:redis_request_count) }
+        end
+
+        def read_bytes_key
+          strong_memoize(:read_bytes_key) { build_key(:redis_read_bytes) }
+        end
+
+        def write_bytes_key
+          strong_memoize(:write_bytes_key) { build_key(:redis_write_bytes) }
+        end
+
+        def call_duration_key
+          strong_memoize(:call_duration_key) { build_key(:redis_call_duration) }
+        end
+
+        def call_details_key
+          strong_memoize(:call_details_key) { build_key(:redis_call_details) }
+        end
+
+        def build_key(namespace)
+          "#{storage_key}_#{namespace}"
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/instrumentation/redis_interceptor.rb b/lib/gitlab/instrumentation/redis_interceptor.rb
new file mode 100644
index 00000000000..a36aade59c3
--- /dev/null
+++ b/lib/gitlab/instrumentation/redis_interceptor.rb
@@ -0,0 +1,86 @@
+# frozen_string_literal: true
+
+require 'redis'
+
+module Gitlab
+  module Instrumentation
+    module RedisInterceptor
+      def call(*args, &block)
+        start = Time.now
+        super(*args, &block)
+      ensure
+        duration = (Time.now - start)
+
+        if ::RequestStore.active?
+          instrumentation_class.increment_request_count
+          instrumentation_class.add_duration(duration)
+          instrumentation_class.add_call_details(duration, args)
+        end
+      end
+
+      def write(command)
+        measure_write_size(command) if ::RequestStore.active?
+        super
+      end
+
+      def read
+        result = super
+        measure_read_size(result) if ::RequestStore.active?
+        result
+      end
+
+      private
+
+      def measure_write_size(command)
+        size = 0
+
+        # Mimic what happens in
+        # https://github.com/redis/redis-rb/blob/f597f21a6b954b685cf939febbc638f6c803e3a7/lib/redis/connection/command_helper.rb#L8.
+        # This count is an approximation that omits the Redis protocol overhead
+        # of type prefixes, length prefixes and line endings.
+        command.each do |x|
+          size += begin
+            if x.is_a? Array
+              x.inject(0) { |sum, y| sum + y.to_s.bytesize }
+            else
+              x.to_s.bytesize
+            end
+          end
+        end
+
+        instrumentation_class.increment_write_bytes(size)
+      end
+
+      def measure_read_size(result)
+        # The Connection::Ruby#read class can return one of four types of results from read:
+        # https://github.com/redis/redis-rb/blob/f597f21a6b954b685cf939febbc638f6c803e3a7/lib/redis/connection/ruby.rb#L406
+        #
+        # 1. Error (exception, will not reach this line)
+        # 2. Status (string)
+        # 3. Integer (will be converted to string by to_s.bytesize and thrown away)
+        # 4. "Binary" string (i.e. may contain zero byte)
+        # 5. Array of binary string
+
+        if result.is_a? Array
+          # Redis can return nested arrays, e.g. from XRANGE or GEOPOS, so we use recursion here.
+          result.each { |x| measure_read_size(x) }
+        else
+          # This count is an approximation that omits the Redis protocol overhead
+          # of type prefixes, length prefixes and line endings.
+          instrumentation_class.increment_read_bytes(result.to_s.bytesize)
+        end
+      end
+
+      # That's required so it knows which GitLab Redis instance
+      # it's interacting with in order to categorize accordingly.
+      #
+      def instrumentation_class
+        @options[:instrumentation_class] # rubocop:disable Gitlab/ModuleWithInstanceVariables
+      end
+    end
+  end
+end
+
+class ::Redis::Client
+  prepend ::Gitlab::Instrumentation::RedisInterceptor
+end
diff --git a/lib/gitlab/instrumentation/redis_payload.rb b/lib/gitlab/instrumentation/redis_payload.rb
new file mode 100644
index 00000000000..69aafffd124
--- /dev/null
+++ b/lib/gitlab/instrumentation/redis_payload.rb
@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Instrumentation
+    module RedisPayload
+      include ::Gitlab::Utils::StrongMemoize
+
+      # Fetches payload keys from the lazy payload (this avoids
+      # unnecessary processing of the values).
+      def known_payload_keys
+        to_lazy_payload.keys
+      end
+
+      def payload
+        to_lazy_payload.transform_values do |value|
+          result = value.call
+          result if result > 0
+        end.compact
+      end
+
+      private
+
+      def to_lazy_payload
+        strong_memoize(:to_lazy_payload) do
+          key_prefix = storage_key ? "redis_#{storage_key}" : 'redis'
+
+          {
+            "#{key_prefix}_calls": -> { get_request_count },
+            "#{key_prefix}_duration_s": -> { query_time },
+            "#{key_prefix}_read_bytes": -> { read_bytes },
+            "#{key_prefix}_write_bytes": -> { write_bytes }
+          }.symbolize_keys
+        end
+      end
+    end
+  end
+end