Adds inter-service OpenTracing propagation

This change allows the GitLab rails and sidekiq components to receive
tracing spans from upstream services such as Workhorse and pass these
spans on to downstream services including Gitaly and Sidekiq.

This change will also emit traces for incoming and outgoing requests
using the propagated trace information. This will allow operators and
engineers to view traces across the Workhorse, GitLab Rails, Sidekiq and
Gitaly components.

Additional intra-service instrumentation will be added in future
changes.

7 files changed, 241 insertions, 1 deletions

diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb
index 8bf8a3b53cd..85afbd85fe6 100644
--- a/lib/gitlab/gitaly_client.rb
+++ b/lib/gitlab/gitaly_client.rb
@@ -52,11 +52,18 @@ module Gitlab
           klass = stub_class(name)
           addr = stub_address(storage)
           creds = stub_creds(storage)
-          klass.new(addr, creds)
+          klass.new(addr, creds, interceptors: interceptors)
         end
       end
     end
 
+    def self.interceptors
+      return [] unless Gitlab::Tracing.enabled?
+
+      [Gitlab::Tracing::GRPCInterceptor.instance]
+    end
+    private_class_method :interceptors
+
     def self.stub_cert_paths
       cert_paths = Dir["#{OpenSSL::X509::DEFAULT_CERT_DIR}/*"]
       cert_paths << OpenSSL::X509::DEFAULT_CERT_FILE if File.exist? OpenSSL::X509::DEFAULT_CERT_FILE
diff --git a/lib/gitlab/tracing/common.rb b/lib/gitlab/tracing/common.rb
new file mode 100644
index 00000000000..5e2b12e3f90
--- /dev/null
+++ b/lib/gitlab/tracing/common.rb
@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Tracing
+    module Common
+      def tracer
+        OpenTracing.global_tracer
+      end
+
+      # Convience method for running a block with a span
+      def in_tracing_span(operation_name:, tags:, child_of: nil)
+        scope = tracer.start_active_span(
+          operation_name,
+          child_of: child_of,
+          tags: tags
+        )
+        span = scope.span
+
+        # Add correlation details to the span if we have them
+        correlation_id = Gitlab::CorrelationId.current_id
+        if correlation_id
+          span.set_tag('correlation_id', correlation_id)
+        end
+
+        begin
+          yield span
+        rescue => e
+          log_exception_on_span(span, e)
+          raise e
+        ensure
+          scope.close
+        end
+      end
+
+      def log_exception_on_span(span, exception)
+        span.set_tag('error', true)
+        span.log_kv(kv_tags_for_exception(exception))
+      end
+
+      def kv_tags_for_exception(exception)
+        case exception
+        when Exception
+          {
+            'event':      'error',
+            'error.kind': exception.class.to_s,
+            'message':    Gitlab::UrlSanitizer.sanitize(exception.message),
+            'stack':      exception.backtrace.join("\n")
+          }
+        else
+          {
+            'event':        'error',
+            'error.kind':   exception.class.to_s,
+            'error.object': Gitlab::UrlSanitizer.sanitize(exception.to_s)
+          }
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/tracing/grpc_interceptor.rb b/lib/gitlab/tracing/grpc_interceptor.rb
new file mode 100644
index 00000000000..6c2aab73125
--- /dev/null
+++ b/lib/gitlab/tracing/grpc_interceptor.rb
@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+require 'opentracing'
+require 'grpc'
+
+module Gitlab
+  module Tracing
+    class GRPCInterceptor < GRPC::ClientInterceptor
+      include Common
+      include Singleton
+
+      def request_response(request:, call:, method:, metadata:)
+        wrap_with_tracing(method, 'unary', metadata) do
+          yield
+        end
+      end
+
+      def client_streamer(requests:, call:, method:, metadata:)
+        wrap_with_tracing(method, 'client_stream', metadata) do
+          yield
+        end
+      end
+
+      def server_streamer(request:, call:, method:, metadata:)
+        wrap_with_tracing(method, 'server_stream', metadata) do
+          yield
+        end
+      end
+
+      def bidi_streamer(requests:, call:, method:, metadata:)
+        wrap_with_tracing(method, 'bidi_stream', metadata) do
+          yield
+        end
+      end
+
+      private
+
+      def wrap_with_tracing(method, grpc_type, metadata)
+        tags = {
+          'component' =>  'grpc',
+          'span.kind' =>  'client',
+          'grpc.method' => method,
+          'grpc.type' =>   grpc_type
+        }
+
+        in_tracing_span(operation_name: "grpc:#{method}", tags: tags) do |span|
+          OpenTracing.inject(span.context, OpenTracing::FORMAT_TEXT_MAP, metadata)
+
+          yield
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/tracing/rack_middleware.rb b/lib/gitlab/tracing/rack_middleware.rb
new file mode 100644
index 00000000000..e6a31293f7b
--- /dev/null
+++ b/lib/gitlab/tracing/rack_middleware.rb
@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+require 'opentracing'
+
+module Gitlab
+  module Tracing
+    class RackMiddleware
+      include Common
+
+      REQUEST_METHOD = 'REQUEST_METHOD'
+
+      def initialize(app)
+        @app = app
+      end
+
+      def call(env)
+        method = env[REQUEST_METHOD]
+
+        context = tracer.extract(OpenTracing::FORMAT_RACK, env)
+        tags = {
+          'component' =>   'rack',
+          'span.kind' =>   'server',
+          'http.method' => method,
+          'http.url' =>    self.class.build_sanitized_url_from_env(env)
+        }
+
+        in_tracing_span(operation_name: "http:#{method}", child_of: context, tags: tags) do |span|
+          @app.call(env).tap do |status_code, _headers, _body|
+            span.set_tag('http.status_code', status_code)
+          end
+        end
+      end
+
+      # Generate a sanitized (safe) request URL from the rack environment
+      def self.build_sanitized_url_from_env(env)
+        request = ActionDispatch::Request.new(env)
+
+        original_url = request.original_url
+        uri = URI.parse(original_url)
+        uri.query = request.filtered_parameters.to_query if uri.query.present?
+
+        uri.to_s
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/tracing/sidekiq/client_middleware.rb b/lib/gitlab/tracing/sidekiq/client_middleware.rb
new file mode 100644
index 00000000000..2b71c1ea21e
--- /dev/null
+++ b/lib/gitlab/tracing/sidekiq/client_middleware.rb
@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+require 'opentracing'
+
+module Gitlab
+  module Tracing
+    module Sidekiq
+      class ClientMiddleware
+        include SidekiqCommon
+
+        SPAN_KIND = 'client'
+
+        def call(worker_class, job, queue, redis_pool)
+          in_tracing_span(
+            operation_name: "sidekiq:#{job['class']}",
+            tags: tags_from_job(job, SPAN_KIND)) do |span|
+            # Inject the details directly into the job
+            tracer.inject(span.context, OpenTracing::FORMAT_TEXT_MAP, job)
+
+            yield
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/tracing/sidekiq/server_middleware.rb b/lib/gitlab/tracing/sidekiq/server_middleware.rb
new file mode 100644
index 00000000000..5b43c4310e6
--- /dev/null
+++ b/lib/gitlab/tracing/sidekiq/server_middleware.rb
@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+require 'opentracing'
+
+module Gitlab
+  module Tracing
+    module Sidekiq
+      class ServerMiddleware
+        include SidekiqCommon
+
+        SPAN_KIND = 'server'
+
+        def call(worker, job, queue)
+          context = tracer.extract(OpenTracing::FORMAT_TEXT_MAP, job)
+
+          in_tracing_span(
+            operation_name: "sidekiq:#{job['class']}",
+            child_of: context,
+            tags: tags_from_job(job, SPAN_KIND)) do |span|
+            yield
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/tracing/sidekiq/sidekiq_common.rb b/lib/gitlab/tracing/sidekiq/sidekiq_common.rb
new file mode 100644
index 00000000000..a911a29d773
--- /dev/null
+++ b/lib/gitlab/tracing/sidekiq/sidekiq_common.rb
@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Tracing
+    module Sidekiq
+      module SidekiqCommon
+        include Gitlab::Tracing::Common
+
+        def tags_from_job(job, kind)
+          {
+            'component' =>     'sidekiq',
+            'span.kind' =>     kind,
+            'sidekiq.queue' => job['queue'],
+            'sidekiq.jid' =>   job['jid'],
+            'sidekiq.retry' => job['retry'].to_s,
+            'sidekiq.args' =>  job['args']&.join(", ")
+          }
+        end
+      end
+    end
+  end
+end