Merge branch 'an-spawn-span-propagation' into 'master'

Support distributed tracing in child processes

See merge request gitlab-org/gitaly!1085

18 files changed, 479 insertions, 59 deletions

diff --git a/.gitignore b/.gitignore
index 0c41b8995..eff022ea3 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,6 +1,8 @@
 /_build
 /gitaly
-/**/gitaly-ssh
+cmd/gitaly-ssh/gitaly-ssh
+/gitaly-ssh
+**/testdata/gitaly-libexec/
 /*.deb
 /_support/package/bin
 /_support/bin
diff --git a/changelogs/unreleased/an-spawn-span-propagation.yml b/changelogs/unreleased/an-spawn-span-propagation.yml
new file mode 100644
index 000000000..43dd9ff75
--- /dev/null
+++ b/changelogs/unreleased/an-spawn-span-propagation.yml
@@ -0,0 +1,5 @@
+---
+title: Support distributed tracing in child processes
+merge_request: 1085
+author:
+type: other
diff --git a/cmd/gitaly-ssh/main.go b/cmd/gitaly-ssh/main.go
index 9f4fdbc12..4049cc47b 100644
--- a/cmd/gitaly-ssh/main.go
+++ b/cmd/gitaly-ssh/main.go
@@ -1,16 +1,21 @@
 package main
 
 import (
+	"context"
 	"fmt"
 	"log"
 	"os"
 
+	grpc_middleware "github.com/grpc-ecosystem/go-grpc-middleware"
 	gitalyauth "gitlab.com/gitlab-org/gitaly/auth"
 	"gitlab.com/gitlab-org/gitaly/client"
+	grpccorrelation "gitlab.com/gitlab-org/labkit/correlation/grpc"
+	"gitlab.com/gitlab-org/labkit/tracing"
+	grpctracing "gitlab.com/gitlab-org/labkit/tracing/grpc"
 	"google.golang.org/grpc"
 )
 
-type packFn func(_ *grpc.ClientConn, _ string) (int32, error)
+type packFn func(_ context.Context, _ *grpc.ClientConn, _ string) (int32, error)
 
 // GITALY_ADDRESS="tcp://1.2.3.4:9999" or "unix:/var/run/gitaly.sock"
 // GITALY_TOKEN="foobar1234"
@@ -24,8 +29,9 @@ func main() {
 		log.Fatalf("invalid number of arguments, expected at least 1, got %d", n-1)
 	}
 
+	command := os.Args[1]
 	var packer packFn
-	switch os.Args[1] {
+	switch command {
 	case "upload-pack":
 		packer = uploadPack
 	case "receive-pack":
@@ -33,27 +39,47 @@ func main() {
 	case "upload-archive":
 		packer = uploadArchive
 	default:
-		log.Fatalf("invalid pack command: %q", os.Args[1])
+		log.Fatalf("invalid pack command: %q", command)
 	}
 
-	if wd := os.Getenv("GITALY_WD"); wd != "" {
-		if err := os.Chdir(wd); err != nil {
-			log.Fatalf("change to : %v", err)
+	gitalyWorkingDir := os.Getenv("GITALY_WD")
+	gitalyAddress := os.Getenv("GITALY_ADDRESS")
+	gitalyPayload := os.Getenv("GITALY_PAYLOAD")
+
+	code, err := run(packer, gitalyWorkingDir, gitalyAddress, gitalyPayload)
+	if err != nil {
+		log.Printf("%s: %v", command, err)
+	}
+
+	os.Exit(code)
+}
+
+func run(packer packFn, gitalyWorkingDir string, gitalyAddress string, gitalyPayload string) (int, error) {
+	// Configure distributed tracing
+	closer := tracing.Initialize(tracing.WithServiceName("gitaly-ssh"))
+	defer closer.Close()
+
+	ctx, finished := tracing.ExtractFromEnv(context.Background())
+	defer finished()
+
+	if gitalyWorkingDir != "" {
+		if err := os.Chdir(gitalyWorkingDir); err != nil {
+			return 1, fmt.Errorf("unable to chdir to %v", gitalyWorkingDir)
 		}
 	}
 
-	conn, err := getConnection(os.Getenv("GITALY_ADDRESS"))
+	conn, err := getConnection(gitalyAddress)
 	if err != nil {
-		log.Fatalf("%s: %v", os.Args[1], err)
+		return 1, err
 	}
 	defer conn.Close()
 
-	code, err := packer(conn, os.Getenv("GITALY_PAYLOAD"))
+	code, err := packer(ctx, conn, gitalyPayload)
 	if err != nil {
-		log.Fatalf("%s: %v", os.Args[1], err)
+		return 1, err
 	}
 
-	os.Exit(int(code))
+	return int(code), nil
 }
 
 func getConnection(url string) (*grpc.ClientConn, error) {
@@ -70,5 +96,18 @@ func dialOpts() []grpc.DialOption {
 		connOpts = append(connOpts, grpc.WithPerRPCCredentials(gitalyauth.RPCCredentials(token)))
 	}
 
+	// Add grpc client interceptors
+	connOpts = append(connOpts, grpc.WithStreamInterceptor(
+		grpc_middleware.ChainStreamClient(
+			grpctracing.StreamClientTracingInterceptor(),         // Tracing
+			grpccorrelation.StreamClientCorrelationInterceptor(), // Correlation
+		)),
+
+		grpc.WithUnaryInterceptor(
+			grpc_middleware.ChainUnaryClient(
+				grpctracing.UnaryClientTracingInterceptor(),         // Tracing
+				grpccorrelation.UnaryClientCorrelationInterceptor(), // Correlation
+			)))
+
 	return connOpts
 }
diff --git a/cmd/gitaly-ssh/main_test.go b/cmd/gitaly-ssh/main_test.go
new file mode 100644
index 000000000..97073794e
--- /dev/null
+++ b/cmd/gitaly-ssh/main_test.go
@@ -0,0 +1,103 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"gitlab.com/gitlab-org/gitaly/internal/testhelper"
+	"google.golang.org/grpc"
+)
+
+func TestRun(t *testing.T) {
+	var successPacker packFn = func(_ context.Context, _ *grpc.ClientConn, _ string) (int32, error) { return 0, nil }
+	var exitCodePacker packFn = func(_ context.Context, _ *grpc.ClientConn, _ string) (int32, error) { return 123, nil }
+	var errorPacker packFn = func(_ context.Context, _ *grpc.ClientConn, _ string) (int32, error) { return 1, fmt.Errorf("fail") }
+
+	gitalyTCPAddress := "tcp://localhost:9999"
+	gitalyUnixAddress := fmt.Sprintf("unix://%s", testhelper.GetTemporaryGitalySocketFileName())
+
+	tests := []struct {
+		name          string
+		workingDir    string
+		gitalyAddress string
+		packer        packFn
+		wantCode      int
+		wantErr       bool
+	}{
+		{
+			name:          "trivial_tcp",
+			packer:        successPacker,
+			gitalyAddress: gitalyTCPAddress,
+			wantCode:      0,
+			wantErr:       false,
+		},
+		{
+			name:          "trivial_unix",
+			packer:        successPacker,
+			gitalyAddress: gitalyUnixAddress,
+			wantCode:      0,
+			wantErr:       false,
+		},
+		{
+			name:          "with_working_dir",
+			workingDir:    os.TempDir(),
+			gitalyAddress: gitalyTCPAddress,
+			packer:        successPacker,
+			wantCode:      0,
+			wantErr:       false,
+		},
+		{
+			name:          "incorrect_working_dir",
+			workingDir:    "directory_does_not_exist",
+			gitalyAddress: gitalyTCPAddress,
+			packer:        successPacker,
+			wantCode:      1,
+			wantErr:       true,
+		},
+		{
+			name:          "empty_gitaly_address",
+			gitalyAddress: "",
+			packer:        successPacker,
+			wantCode:      1,
+			wantErr:       true,
+		},
+		{
+			name:          "invalid_gitaly_address",
+			gitalyAddress: "invalid_gitaly_address",
+			packer:        successPacker,
+			wantCode:      1,
+			wantErr:       true,
+		},
+		{
+			name:          "exit_code",
+			gitalyAddress: gitalyTCPAddress,
+			packer:        exitCodePacker,
+			wantCode:      123,
+			wantErr:       false,
+		},
+		{
+			name:          "error",
+			gitalyAddress: gitalyTCPAddress,
+			packer:        errorPacker,
+			wantCode:      1,
+			wantErr:       true,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			gotCode, err := run(tt.packer, tt.workingDir, tt.gitalyAddress, "{}")
+			if tt.wantErr {
+				assert.Error(t, err)
+			} else {
+				assert.NoError(t, err)
+			}
+
+			assert.Equal(t, tt.wantCode, gotCode)
+		})
+	}
+}
diff --git a/cmd/gitaly-ssh/receive_pack.go b/cmd/gitaly-ssh/receive_pack.go
index cb4f404ed..910eb7628 100644
--- a/cmd/gitaly-ssh/receive_pack.go
+++ b/cmd/gitaly-ssh/receive_pack.go
@@ -11,13 +11,13 @@ import (
 	"google.golang.org/grpc"
 )
 
-func receivePack(conn *grpc.ClientConn, req string) (int32, error) {
+func receivePack(ctx context.Context, conn *grpc.ClientConn, req string) (int32, error) {
 	var request gitalypb.SSHReceivePackRequest
 	if err := jsonpb.UnmarshalString(req, &request); err != nil {
 		return 0, fmt.Errorf("json unmarshal: %v", err)
 	}
 
-	ctx, cancel := context.WithCancel(context.Background())
+	ctx, cancel := context.WithCancel(ctx)
 	defer cancel()
 
 	return client.ReceivePack(ctx, conn, os.Stdin, os.Stdout, os.Stderr, &request)
diff --git a/cmd/gitaly-ssh/upload_archive.go b/cmd/gitaly-ssh/upload_archive.go
index b7bb86466..104227d36 100644
--- a/cmd/gitaly-ssh/upload_archive.go
+++ b/cmd/gitaly-ssh/upload_archive.go
@@ -11,13 +11,13 @@ import (
 	"google.golang.org/grpc"
 )
 
-func uploadArchive(conn *grpc.ClientConn, req string) (int32, error) {
+func uploadArchive(ctx context.Context, conn *grpc.ClientConn, req string) (int32, error) {
 	var request gitalypb.SSHUploadArchiveRequest
 	if err := jsonpb.UnmarshalString(req, &request); err != nil {
 		return 0, fmt.Errorf("json unmarshal: %v", err)
 	}
 
-	ctx, cancel := context.WithCancel(context.Background())
+	ctx, cancel := context.WithCancel(ctx)
 	defer cancel()
 
 	return client.UploadArchive(ctx, conn, os.Stdin, os.Stdout, os.Stderr, &request)
diff --git a/cmd/gitaly-ssh/upload_pack.go b/cmd/gitaly-ssh/upload_pack.go
index d883260e1..98c688758 100644
--- a/cmd/gitaly-ssh/upload_pack.go
+++ b/cmd/gitaly-ssh/upload_pack.go
@@ -11,13 +11,13 @@ import (
 	"google.golang.org/grpc"
 )
 
-func uploadPack(conn *grpc.ClientConn, req string) (int32, error) {
+func uploadPack(ctx context.Context, conn *grpc.ClientConn, req string) (int32, error) {
 	var request gitalypb.SSHUploadPackRequest
 	if err := jsonpb.UnmarshalString(req, &request); err != nil {
 		return 0, fmt.Errorf("json unmarshal: %v", err)
 	}
 
-	ctx, cancel := context.WithCancel(context.Background())
+	ctx, cancel := context.WithCancel(ctx)
 	defer cancel()
 
 	return client.UploadPack(ctx, conn, os.Stdin, os.Stdout, os.Stderr, &request)
diff --git a/internal/service/repository/fork.go b/internal/service/repository/fork.go
index fedef0227..ce480e7d1 100644
--- a/internal/service/repository/fork.go
+++ b/internal/service/repository/fork.go
@@ -11,12 +11,15 @@ import (
 	"gitlab.com/gitlab-org/gitaly/internal/config"
 	"gitlab.com/gitlab-org/gitaly/internal/git"
 	"gitlab.com/gitlab-org/gitaly/internal/helper"
+	"gitlab.com/gitlab-org/labkit/tracing"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"
 )
 
 const gitalyInternalURL = "ssh://gitaly/internal.git"
 
+var envInjector = tracing.NewEnvInjector()
+
 func (s *server) CreateFork(ctx context.Context, req *gitalypb.CreateForkRequest) (*gitalypb.CreateForkResponse, error) {
 	targetRepository := req.Repository
 	sourceRepository := req.SourceRepository
@@ -73,6 +76,8 @@ func (s *server) CreateFork(ctx context.Context, req *gitalypb.CreateForkRequest
 		fmt.Sprintf("GITALY_TOKEN=%s", sourceRepositoryGitalyToken),
 		fmt.Sprintf("GIT_SSH_COMMAND=%s upload-pack", gitalySSHPath),
 	}
+	env = envInjector(ctx, env)
+
 	args := []string{
 		"clone",
 		"--bare",
diff --git a/internal/supervisor/supervisor.go b/internal/supervisor/supervisor.go
index 72a0bd19a..e93838256 100644
--- a/internal/supervisor/supervisor.go
+++ b/internal/supervisor/supervisor.go
@@ -1,6 +1,7 @@
 package supervisor
 
 import (
+	"context"
 	"fmt"
 	"io"
 	"os/exec"
@@ -10,6 +11,7 @@ import (
 	"github.com/kelseyhightower/envconfig"
 	"github.com/prometheus/client_golang/prometheus"
 	log "github.com/sirupsen/logrus"
+	"gitlab.com/gitlab-org/labkit/tracing"
 )
 
 // Config holds configuration for the circuit breaker of the respawn loop.
@@ -31,7 +33,8 @@ var (
 		[]string{"name"},
 	)
 
-	config Config
+	config      Config
+	envInjector = tracing.NewEnvInjector()
 )
 
 func init() {
@@ -69,7 +72,7 @@ func New(name string, env []string, args []string, dir string, memoryThreshold i
 		memoryThreshold: memoryThreshold,
 		events:          events,
 		healthCheck:     healthCheck,
-		env:             env,
+		env:             envInjector(context.Background(), env),
 		args:            args,
 		dir:             dir,
 		shutdown:        make(chan struct{}),
diff --git a/vendor/gitlab.com/gitlab-org/labkit/correlation/inbound_http.go b/vendor/gitlab.com/gitlab-org/labkit/correlation/inbound_http.go
index b7a297c3d..5090f1f85 100644
--- a/vendor/gitlab.com/gitlab-org/labkit/correlation/inbound_http.go
+++ b/vendor/gitlab.com/gitlab-org/labkit/correlation/inbound_http.go
@@ -11,7 +11,6 @@ import (
 // Whether the Correlation-ID is generated or propagated, once inside this handler the request context
 // will have a Correlation-ID associated with it.
 func InjectCorrelationID(h http.Handler, opts ...InboundHandlerOption) http.Handler {
-	// Currently we don't use any of the options available
 	config := applyInboundHandlerOptions(opts)
 
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
@@ -26,6 +25,10 @@ func InjectCorrelationID(h http.Handler, opts ...InboundHandlerOption) http.Hand
 			correlationID = generateRandomCorrelationIDWithFallback(r)
 		}
 
+		if config.sendResponseHeader {
+			setResponseHeader(w, correlationID)
+		}
+
 		ctx := ContextWithCorrelation(parent, correlationID)
 		h.ServeHTTP(w, r.WithContext(ctx))
 	})
@@ -34,3 +37,7 @@ func InjectCorrelationID(h http.Handler, opts ...InboundHandlerOption) http.Hand
 func extractFromRequest(r *http.Request) string {
 	return r.Header.Get(propagationHeader)
 }
+
+func setResponseHeader(w http.ResponseWriter, correlationID string) {
+	w.Header().Set(propagationHeader, correlationID)
+}
diff --git a/vendor/gitlab.com/gitlab-org/labkit/correlation/inbound_http_options.go b/vendor/gitlab.com/gitlab-org/labkit/correlation/inbound_http_options.go
index 2bad8e46c..253f663cb 100644
--- a/vendor/gitlab.com/gitlab-org/labkit/correlation/inbound_http_options.go
+++ b/vendor/gitlab.com/gitlab-org/labkit/correlation/inbound_http_options.go
@@ -2,7 +2,8 @@ package correlation
 
 // The configuration for InjectCorrelationID
 type inboundHandlerConfig struct {
-	propagation bool
+	propagation        bool
+	sendResponseHeader bool
 }
 
 // InboundHandlerOption will configure a correlation handler
@@ -29,3 +30,11 @@ func WithPropagation() InboundHandlerOption {
 		config.propagation = true
 	}
 }
+
+// WithSetResponseHeader will configure the handler to set the correlation_id
+// in the http response headers
+func WithSetResponseHeader() InboundHandlerOption {
+	return func(config *inboundHandlerConfig) {
+		config.sendResponseHeader = true
+	}
+}
diff --git a/vendor/gitlab.com/gitlab-org/labkit/tracing/doc.go b/vendor/gitlab.com/gitlab-org/labkit/tracing/doc.go
new file mode 100644
index 000000000..245b6ab94
--- /dev/null
+++ b/vendor/gitlab.com/gitlab-org/labkit/tracing/doc.go
@@ -0,0 +1,98 @@
+/*
+Package tracing is the primary entrypoint into LabKit's distributed tracing functionality.
+
+(This documentation assumes some minimal knowledge of Distributed Tracing, and uses
+tracing terminology without providing definitions. Please review
+https://opentracing.io/docs/overview/what-is-tracing/ for an broad overview of distributed
+tracing if you are not familiar with the technology)
+
+Internally the `tracing` package relies on Opentracing, but avoids leaking this abstraction.
+In theory, LabKit could replace Opentracing with another distributed tracing interface, such
+as Zipkin or OpenCensus, without needing to make changes to the application (other than vendoring
+in a new version of LabKit, of course).
+
+This design decision is deliberate: the package should not leak the underlying tracing implementation.
+
+The package provides three primary exports:
+
+* `tracing.Initialize()` for initializing the global tracer using the `GITLAB_TRACING` environment variable.
+* An HTTP Handler middleware, `tracing.Handler()`, for instrumenting incoming HTTP requests.
+* An HTTP RoundTripper, `tracing.NewRoundTripper()` for instrumenting outbound HTTP requests to other services.
+
+The provided example in `example_test.go` demonstrates usage of both the HTTP Middleware and the HTTP RoundTripper.
+
+*Initializing the global tracer*
+
+Opentracing makes use of a global tracer. Opentracing ships with a default NoOp tracer which does
+nothing at all. This is always configured, meaning that, without initialization, Opentracing does nothing and
+has a very low overhead.
+
+LabKit's tracing is configured through an environment variable, `GITLAB_TRACING`. This environment variable contains
+a "connection string"-like configuration, such as:
+
+* `opentracing://jaeger?udp_endpoint=localhost:6831`
+* `opentracing://datadog`
+* `opentracing://lightstep`
+
+The parameters for these connection-strings are implementation specific.
+
+This configuration is identical to the one used to configure GitLab's ruby tracing libraries in the `Gitlab::Tracing`
+package. Having a consistent configuration makes it easy to configure multiple processes at the same time. For example,
+in GitLab Development Kit, tracing can be configured with a single environment variable, `GITLAB_TRACING=... gdk run`,
+since `GITLAB_TRACING` will configure Workhorse (written in Go), Gitaly (written in Go) and GitLab's rails components,
+using the same configuration.
+
+*Compiling applications with Tracing support*
+
+Go's Opentracing interface does not allow tracing implementations to be loaded dynamically; implementations need to be
+compiled into the application. With LabKit, this is done conditionally, using build tags. Two build tags need to be
+specified:
+
+* `tracer_static` - this compiles in the static plugin registry support
+* `tracer_static_[DRIVER_NAME]` - this compile in support for the given driver.
+
+For example, to compile support for Jaeger, compile your Go app with `tracer_static,tracer_static_jaeger`
+
+Note that multiple (or all) drivers can be compiled in alongside one another: using the tags:
+`tracer_static,tracer_static_jaeger,tracer_static_lightstep,tracer_static_datadog`
+
+If the `GITLAB_TRACING` environment variable references an unknown or unregistered driver, it will log a message
+and continue without tracing. This is a deliberate decision: the risk of bringing down a cluster during a rollout
+with a misconfigured tracer configuration is greater than the risk of an operator loosing some time because
+their application was not compiled with the correct tracers.
+
+*Using the HTTP Handler middleware to instrument incoming HTTP requests*
+
+When an incoming HTTP request arrives on the server, it may already include Distributed Tracing headers,
+propagated from an upstream service.
+
+The tracing middleware will attempt to extract the tracing information from the headers (the exact headers used are
+tracing implementation specific), set up a span and pass the information through the request context.
+
+It is up to the Opentracing implementation to decide whether the span will be sent to the tracing infrastructure.
+This will be implementation-specific, but generally relies on server load, sampler configuration, whether an
+error occurred, whether certain spans took an anomalous amount of time, etc.
+
+*Using the  HTTP RoundTripper to instrument outgoing HTTP requests*
+
+The RoundTripper should be added to the HTTP client RoundTripper stack (see the example). When an outbound
+HTTP request is sent from the HTTP client, the RoundTripper will determine whether there is an active span
+and if so, will inject headers into the outgoing HTTP request identifying the span. The details of these
+headers is implementation specific.
+
+It is important to ensure that the context is passed into the outgoing request, using `req.WithContext(ctx)`
+so that the correct span information can be injected into the request headers.
+
+*Propagating tracing information to child processes*
+
+Sometimes we want a trace to continue from a parent process to a spawned child process. For this,
+the tracing package provides `tracing.NewEnvInjector()` and `tracing.ExtractFromEnv()`, for the
+parent and child processes respectively.
+
+NewEnvInjector() will configure a []string array of environment variables, ensuring they have the
+correct tracing configuration and any trace and span identifiers. NewEnvInjector() should be called
+in the child process and will extract the trace and span information from the environment.
+
+Please review the examples in the godocs for details of how to implement both approaches.
+*/
+package tracing
diff --git a/vendor/gitlab.com/gitlab-org/labkit/tracing/env_extractor.go b/vendor/gitlab.com/gitlab-org/labkit/tracing/env_extractor.go
new file mode 100644
index 000000000..2b92a8ab8
--- /dev/null
+++ b/vendor/gitlab.com/gitlab-org/labkit/tracing/env_extractor.go
@@ -0,0 +1,63 @@
+package tracing
+
+import (
+	"context"
+	"os"
+	"strings"
+
+	opentracing "github.com/opentracing/opentracing-go"
+	"github.com/opentracing/opentracing-go/ext"
+	"gitlab.com/gitlab-org/labkit/correlation"
+)
+
+// ExtractFromEnv will extract a span from the environment after it has been passed in
+// from the parent process. Returns a new context, and a defer'able function, which
+// should be called on process termination
+func ExtractFromEnv(ctx context.Context, opts ...ExtractFromEnvOption) (context.Context, func()) {
+	/* config not yet used */ applyExtractFromEnvOptions(opts)
+	tracer := opentracing.GlobalTracer()
+
+	// Extract the Correlation-ID
+	envMap := environAsMap(os.Environ())
+	correlationID := envMap[envCorrelationIDKey]
+	if correlationID != "" {
+		ctx = correlation.ContextWithCorrelation(ctx, correlationID)
+	}
+
+	// Attempt to deserialize tracing identifiers
+	wireContext, err := tracer.Extract(
+		opentracing.TextMap,
+		opentracing.TextMapCarrier(envMap))
+
+	if err != nil {
+		/* Clients could send bad data, in which case we simply ignore it */
+		return ctx, func() {}
+	}
+
+	// Create the span referring to the RPC client if available.
+	// If wireContext == nil, a root span will be created.
+	additionalStartSpanOpts := []opentracing.StartSpanOption{
+		ext.RPCServerOption(wireContext),
+	}
+
+	if correlationID != "" {
+		additionalStartSpanOpts = append(additionalStartSpanOpts, opentracing.Tag{Key: "correlation_id", Value: correlationID})
+	}
+
+	serverSpan := opentracing.StartSpan(
+		"execute",
+		additionalStartSpanOpts...,
+	)
+	ctx = opentracing.ContextWithSpan(ctx, serverSpan)
+
+	return ctx, func() { serverSpan.Finish() }
+}
+
+func environAsMap(env []string) map[string]string {
+	envMap := make(map[string]string, len(env))
+	for _, v := range env {
+		s := strings.SplitN(v, "=", 2)
+		envMap[s[0]] = s[1]
+	}
+	return envMap
+}
diff --git a/vendor/gitlab.com/gitlab-org/labkit/tracing/env_extractor_option.go b/vendor/gitlab.com/gitlab-org/labkit/tracing/env_extractor_option.go
new file mode 100644
index 000000000..e9288c9da
--- /dev/null
+++ b/vendor/gitlab.com/gitlab-org/labkit/tracing/env_extractor_option.go
@@ -0,0 +1,15 @@
+package tracing
+
+type extractFromEnvConfig struct{}
+
+// ExtractFromEnvOption will configure an environment injector
+type ExtractFromEnvOption func(*extractFromEnvConfig)
+
+func applyExtractFromEnvOptions(opts []ExtractFromEnvOption) extractFromEnvConfig {
+	config := extractFromEnvConfig{}
+	for _, v := range opts {
+		v(&config)
+	}
+
+	return config
+}
diff --git a/vendor/gitlab.com/gitlab-org/labkit/tracing/env_injector.go b/vendor/gitlab.com/gitlab-org/labkit/tracing/env_injector.go
new file mode 100644
index 000000000..30d39a1b7
--- /dev/null
+++ b/vendor/gitlab.com/gitlab-org/labkit/tracing/env_injector.go
@@ -0,0 +1,70 @@
+package tracing
+
+import (
+	"context"
+	"fmt"
+	"log"
+	"os"
+	"sort"
+
+	opentracing "github.com/opentracing/opentracing-go"
+	"gitlab.com/gitlab-org/labkit/correlation"
+)
+
+// envCorrelationIDKey is used to pass the current correlation-id over to the child process
+const envCorrelationIDKey = "CORRELATION_ID"
+
+// EnvInjector will inject tracing information into an environment in preparation for
+// spawning a child process. This includes trace and span identifiers, as well
+// as the GITLAB_TRACING configuration. Will gracefully degrade if tracing is
+// not configured, or an active span is not currently available.
+type EnvInjector func(ctx context.Context, env []string) []string
+
+// NewEnvInjector will create a new environment injector
+func NewEnvInjector(opts ...EnvInjectorOption) EnvInjector {
+	/* config not yet used */ applyEnvInjectorOptions(opts)
+
+	return func(ctx context.Context, env []string) []string {
+		envMap := map[string]string{}
+
+		// Pass the Correlation-ID through the environment if set
+		correlationID := correlation.ExtractFromContext(ctx)
+		if correlationID != "" {
+			envMap[envCorrelationIDKey] = correlationID
+		}
+
+		// Also include the GITLAB_TRACING configuration so that
+		// the child process knows how to configure itself
+		v, ok := os.LookupEnv(tracingEnvKey)
+		if ok {
+			envMap[tracingEnvKey] = v
+		}
+
+		span := opentracing.SpanFromContext(ctx)
+		if span == nil {
+			// If no active span, short circuit
+			return appendMapToEnv(env, envMap)
+		}
+
+		carrier := opentracing.TextMapCarrier(envMap)
+		err := span.Tracer().Inject(span.Context(), opentracing.TextMap, carrier)
+
+		if err != nil {
+			log.Printf("tracing span injection failed: %v", err)
+		}
+
+		return appendMapToEnv(env, envMap)
+	}
+}
+
+// appendMapToEnv takes a map of key,value pairs and appends it to an
+// array of environment variable pairs in `K=V` string pairs
+func appendMapToEnv(env []string, envMap map[string]string) []string {
+	additionalItems := []string{}
+	for k, v := range envMap {
+		additionalItems = append(additionalItems, fmt.Sprintf("%s=%s", k, v))
+	}
+
+	sort.Strings(additionalItems)
+	return append(env, additionalItems...)
+}
diff --git a/vendor/gitlab.com/gitlab-org/labkit/tracing/env_injector_option.go b/vendor/gitlab.com/gitlab-org/labkit/tracing/env_injector_option.go
new file mode 100644
index 000000000..e3298a4ee
--- /dev/null
+++ b/vendor/gitlab.com/gitlab-org/labkit/tracing/env_injector_option.go
@@ -0,0 +1,15 @@
+package tracing
+
+type envInjectorConfig struct{}
+
+// EnvInjectorOption will configure an environment injector
+type EnvInjectorOption func(*envInjectorConfig)
+
+func applyEnvInjectorOptions(opts []EnvInjectorOption) envInjectorConfig {
+	config := envInjectorConfig{}
+	for _, v := range opts {
+		v(&config)
+	}
+
+	return config
+}
diff --git a/vendor/gitlab.com/gitlab-org/labkit/tracing/initialization_options.go b/vendor/gitlab.com/gitlab-org/labkit/tracing/initialization_options.go
index 206e815aa..cfb4996dc 100644
--- a/vendor/gitlab.com/gitlab-org/labkit/tracing/initialization_options.go
+++ b/vendor/gitlab.com/gitlab-org/labkit/tracing/initialization_options.go
@@ -5,6 +5,8 @@ import (
 	"path"
 )
 
+const tracingEnvKey = "GITLAB_TRACING"
+
 // The configuration for InjectCorrelationID
 type initializationConfig struct {
 	serviceName      string
@@ -17,7 +19,7 @@ type InitializationOption func(*initializationConfig)
 func applyInitializationOptions(opts []InitializationOption) initializationConfig {
 	config := initializationConfig{
 		serviceName:      path.Base(os.Args[0]),
-		connectionString: os.Getenv("GITLAB_TRACING"),
+		connectionString: os.Getenv(tracingEnvKey),
 	}
 	for _, v := range opts {
 		v(&config)
diff --git a/vendor/vendor.json b/vendor/vendor.json
index 2a5014105..ab9fbaac4 100644
--- a/vendor/vendor.json
+++ b/vendor/vendor.json
@@ -509,67 +509,51 @@
 			"versionExact": "v1.12.0"
 		},
 		{
-			"checksumSHA1": "R6fNN36q3UydLODupK0vdx9h/CY=",
+			"checksumSHA1": "WMOuBgCyclqy+Mqunb0NbykaC4Y=",
 			"path": "gitlab.com/gitlab-org/labkit/correlation",
-			"revision": "9c613f6d8258a1b55701079bf9b358147aea139e",
-			"revisionTime": "2018-12-20T16:08:28Z",
-			"version": "=d-tracing",
-			"versionExact": "d-tracing"
-		},
-		{
-			"checksumSHA1": "R6fNN36q3UydLODupK0vdx9h/CY=",
-			"path": "gitlab.com/gitlab-org/labkit/correlation",
-			"revision": "2a3e1f5415a890402696dcbb2c31b5a79c17b579",
-			"revisionTime": "2019-01-08T10:46:58Z",
-			"version": "master",
+			"revision": "0c3fc7cdd57c57da5ab474aa72b6640d2bdc9ebb",
+			"revisionTime": "2019-02-21T12:25:36Z",
+			"version": "=master",
 			"versionExact": "master"
 		},
 		{
 			"checksumSHA1": "UFBFulprWZHuL9GHhjCKoHXm+Ww=",
 			"path": "gitlab.com/gitlab-org/labkit/correlation/grpc",
-			"revision": "2a3e1f5415a890402696dcbb2c31b5a79c17b579",
-			"revisionTime": "2019-01-08T10:46:58Z",
-			"version": "master",
+			"revision": "0c3fc7cdd57c57da5ab474aa72b6640d2bdc9ebb",
+			"revisionTime": "2019-02-21T12:25:36Z",
+			"version": "=master",
 			"versionExact": "master"
 		},
 		{
-			"checksumSHA1": "UFBFulprWZHuL9GHhjCKoHXm+Ww=",
-			"path": "gitlab.com/gitlab-org/labkit/correlation/grpc",
-			"revision": "3a481e4f5d6cdd19bf9fbbc48a2fe50387b7ae5f",
-			"revisionTime": "2018-12-13T12:30:12Z",
-			"version": "=d-tracing",
-			"versionExact": "d-tracing"
-		},
-		{
-			"checksumSHA1": "JAe9JIwWcJvTUxCbdOPAZxaXIz8=",
+			"checksumSHA1": "6SAh0LdyizW+RICpQglU6WLMhus=",
 			"path": "gitlab.com/gitlab-org/labkit/tracing",
-			"revision": "2a3e1f5415a890402696dcbb2c31b5a79c17b579",
-			"revisionTime": "2019-01-08T10:46:58Z",
-			"version": "master",
+			"revision": "0c3fc7cdd57c57da5ab474aa72b6640d2bdc9ebb",
+			"revisionTime": "2019-02-21T12:25:36Z",
+			"version": "=master",
 			"versionExact": "master"
 		},
 		{
 			"checksumSHA1": "uIvjqXAsMQK/Y5FgWRaGydYGbYs=",
 			"path": "gitlab.com/gitlab-org/labkit/tracing/connstr",
-			"revision": "2a3e1f5415a890402696dcbb2c31b5a79c17b579",
-			"revisionTime": "2019-01-08T10:46:58Z",
-			"version": "master",
+			"revision": "0c3fc7cdd57c57da5ab474aa72b6640d2bdc9ebb",
+			"revisionTime": "2019-02-21T12:25:36Z",
+			"version": "=master",
 			"versionExact": "master"
 		},
 		{
 			"checksumSHA1": "IE38In/zPKpmKqvWAAThyaufQak=",
 			"path": "gitlab.com/gitlab-org/labkit/tracing/grpc",
-			"revision": "2a3e1f5415a890402696dcbb2c31b5a79c17b579",
-			"revisionTime": "2019-01-08T10:46:58Z",
-			"version": "master",
+			"revision": "0c3fc7cdd57c57da5ab474aa72b6640d2bdc9ebb",
+			"revisionTime": "2019-02-21T12:25:36Z",
+			"version": "=master",
 			"versionExact": "master"
 		},
 		{
 			"checksumSHA1": "hB59Es/WTWfBPLSAheQaRyHGSXA=",
 			"path": "gitlab.com/gitlab-org/labkit/tracing/impl",
-			"revision": "2a3e1f5415a890402696dcbb2c31b5a79c17b579",
-			"revisionTime": "2019-01-08T10:46:58Z",
-			"version": "master",
+			"revision": "0c3fc7cdd57c57da5ab474aa72b6640d2bdc9ebb",
+			"revisionTime": "2019-02-21T12:25:36Z",
+			"version": "=master",
 			"versionExact": "master"
 		},
 		{