git: Enable Trace2toon-git-trace2

7 files changed, 158 insertions, 1 deletions

diff --git a/go.mod b/go.mod
index 41f1d3d3f..7a2d38f61 100644
--- a/go.mod
+++ b/go.mod
@@ -38,6 +38,7 @@ require (
 	github.com/stretchr/testify v1.8.0
 	github.com/uber/jaeger-client-go v2.30.0+incompatible
 	gitlab.com/gitlab-org/labkit v1.16.0
+	go.opencensus.io v0.23.0
 	go.uber.org/goleak v1.2.0
 	gocloud.dev v0.26.0
 	golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f
@@ -164,7 +165,6 @@ require (
 	github.com/tklauser/numcpus v0.2.1 // indirect
 	github.com/uber/jaeger-lib v2.4.1+incompatible // indirect
 	github.com/xanzy/ssh-agent v0.3.0 // indirect
-	go.opencensus.io v0.23.0 // indirect
 	go.uber.org/atomic v1.9.0 // indirect
 	golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa // indirect
 	golang.org/x/exp v0.0.0-20200331195152-e8c3332aa8e5 // indirect
diff --git a/internal/command/command.go b/internal/command/command.go
index e3056d593..5abc64547 100644
--- a/internal/command/command.go
+++ b/internal/command/command.go
@@ -89,6 +89,8 @@ var (
 		"GIT_TRACE_PACKET",
 		"GIT_TRACE_PERFORMANCE",
 		"GIT_TRACE_SETUP",
+		"GIT_TRACE2_EVENT",
+		"GIT_TRACE2_PARENT_SID",
 
 		// GIT_EXEC_PATH tells Git where to find its binaries. This must be exported
 		// especially in the case where we use bundled Git executables given that we cannot
diff --git a/internal/git/command_factory.go b/internal/git/command_factory.go
index dae719935..f5dd549b0 100644
--- a/internal/git/command_factory.go
+++ b/internal/git/command_factory.go
@@ -6,8 +6,10 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"strings"
 	"sync"
 
+	"github.com/opentracing/opentracing-go"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/sirupsen/logrus"
 	"gitlab.com/gitlab-org/gitaly/v15/internal/cgroups"
@@ -401,6 +403,23 @@ func (cf *ExecCommandFactory) newCommand(ctx context.Context, repo repository.Gi
 
 	env = append(env, execEnv.EnvironmentVariables...)
 
+	span, ctx := opentracing.StartSpanFromContext(
+		ctx,
+		"git",
+		opentracing.Tag{Key: "args", Value: strings.Join(args, " ")},
+	)
+
+	trace2Env, trace2Finish, err := enableTrace2(ctx, span)
+	if err != nil {
+		return nil, err
+	}
+	env = append(env, trace2Env...)
+
+	finalizer := func(*command.Command) {
+		trace2Finish()
+		span.Finish()
+	}
+
 	cmdGitVersion, err := cf.GitVersion(ctx)
 	if err != nil {
 		return nil, fmt.Errorf("getting Git version: %w", err)
@@ -410,9 +429,11 @@ func (cf *ExecCommandFactory) newCommand(ctx context.Context, repo repository.Gi
 		config.commandOpts,
 		command.WithDir(dir),
 		command.WithEnvironment(env),
+		command.WithSpan(span),
 		command.WithCommandName("git", sc.Subcommand()),
 		command.WithCgroup(cf.cgroupsManager, repo),
 		command.WithCommandGitVersion(cmdGitVersion.String()),
+		command.WithFinalizer(finalizer),
 	)...)
 	if err != nil {
 		return nil, err
diff --git a/internal/git/trace2.go b/internal/git/trace2.go
new file mode 100644
index 000000000..165ec7186
--- /dev/null
+++ b/internal/git/trace2.go
@@ -0,0 +1,107 @@
+package git
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"log"
+	"os"
+	"time"
+
+	"github.com/grpc-ecosystem/go-grpc-middleware/logging/logrus/ctxlogrus"
+	"github.com/opentracing/opentracing-go"
+	"github.com/uber/jaeger-client-go"
+	"gitlab.com/gitlab-org/gitaly/v15/internal/metadata/featureflag"
+	"gitlab.com/gitlab-org/labkit/correlation"
+	"go.opencensus.io/trace"
+)
+
+// trace2Event is used to deserialize the data generated by GIT_TRACE2
+type trace2Event struct {
+	Event   string    `json:"event"`
+	Sid     string    `json:"sid"`
+	Thread  string    `json:"thread"`
+	Time    time.Time `json:"time"`
+	File    string    `json:"file"`
+	Line    int       `json:"line"`
+	AbsTime float64   `json:"t_abs"`
+	Code    int       `json:"code"`
+}
+
+// enableTrace2 will prepare the environment to send Git TRACE2 events to
+// opentracing. It return a slice of environment variables that need to be
+// injected into the Command, and a finish function that needs to be called when
+// the command is done.
+func enableTrace2(ctx context.Context, span opentracing.Span) ([]string, func(), error) {
+	if !isTracingEnabled(ctx, span) {
+		return []string{}, func() {}, nil
+	}
+
+	tmpFile, err := os.CreateTemp("", "gitaly-trace2.")
+	if err != nil {
+		return nil, nil, fmt.Errorf("Trace2 create tempfile: %w", err)
+	}
+
+	env := []string{
+		fmt.Sprintf("GIT_TRACE2_EVENT=%v", tmpFile.Name()),
+		fmt.Sprintf("GIT_TRACE2_PARENT_SID=%s", correlation.ExtractFromContextOrGenerate(ctx)),
+	}
+
+	finish := func() {
+		dec := json.NewDecoder(tmpFile)
+		var ev trace2Event
+
+		for {
+			err := dec.Decode(&ev)
+			if err == nil {
+				span.LogKV(
+					"trace2.event", ev.Event,
+					"trace2.sid", ev.Sid,
+					"trace2.thread", ev.Thread,
+					"trace2.time", ev.Time,
+					"trace2.file", ev.File,
+					"trace2.line", ev.Line,
+					"trace2.t_abs", ev.AbsTime,
+					"trace2.code", ev.Code)
+			} else if errors.Is(err, io.EOF) {
+				break
+			} else {
+				log.Println("the error %w", err)
+				break
+			}
+		}
+
+		if err := tmpFile.Close(); err != nil {
+			ctxlogrus.Extract(ctx).WithError(err).Error("failed to close TRACE2 temp file")
+		}
+		if err := os.Remove(tmpFile.Name()); err != nil {
+			ctxlogrus.Extract(ctx).WithError(err).Error("failed to remove TRACE2 temp file")
+		}
+	}
+
+	return env, finish, nil
+}
+
+func isTracingEnabled(ctx context.Context, span opentracing.Span) bool {
+	// Only when the feature flag is enabled
+	if featureflag.GitTrace2.IsDisabled(ctx) {
+		return false
+	}
+
+	// No tracer registered
+	if _, ok := span.Tracer().(opentracing.NoopTracer); ok {
+		return false
+	}
+
+	// Jaeger is only used in local development
+	if _, ok := span.Tracer().(*jaeger.Tracer); ok {
+		return true
+	}
+
+	// Only enable TRACE2 if context is actively sampled
+	// Taken from https://gitlab.com/gitlab-org/labkit/-/blob/959b71911822d1c6cc1c2ef3bccf844d814066d6/tracing/impl/stackdriver_tracer.go#L77
+	// return trace.FromContext(ctx).SpanContext().IsSampled()
+	return trace.FromContext(ctx).IsRecordingEvents()
+}
diff --git a/internal/git/trace2_test.go b/internal/git/trace2_test.go
new file mode 100644
index 000000000..b8c10be93
--- /dev/null
+++ b/internal/git/trace2_test.go
@@ -0,0 +1,16 @@
+package git
+
+import (
+	"testing"
+)
+
+func TestEnableTrace2(t *testing.T) {
+	// ctx := testhelper.Context(t)
+
+	// buff := &bytes.Buffer{}
+	// cmd, err := New(ctx, exec.Command("/usr/bin/env"), nil, buff, nil)
+	// require.NoError(t, err)
+
+	// require.Len(t, cmd.cmd.ExtraFiles, 1)
+	// require.Contains(t, cmd.cmd.Env, "GIT_TRACE2_EVENT=3")
+}
diff --git a/internal/metadata/featureflag/ff_git_trace2.go b/internal/metadata/featureflag/ff_git_trace2.go
new file mode 100644
index 000000000..9f50710cd
--- /dev/null
+++ b/internal/metadata/featureflag/ff_git_trace2.go
@@ -0,0 +1,9 @@
+package featureflag
+
+// GitTrace2 enables TRACE2 on git calls
+var GitTrace2 = NewFeatureFlag(
+	"git_trace2",
+	"v15.5",
+	"https://gitlab.com/gitlab-org/gitaly/-/issues/TODO",
+	false,
+)
diff --git a/internal/testhelper/testhelper.go b/internal/testhelper/testhelper.go
index 02440e7f0..470b00342 100644
--- a/internal/testhelper/testhelper.go
+++ b/internal/testhelper/testhelper.go
@@ -199,6 +199,8 @@ func ContextWithoutCancel(opts ...ContextOpt) context.Context {
 	// PraefectGeneratedReplicaPaths affects many tests as it changes the repository creation logic.
 	// Randomly enable the flag to exercise both paths to some extent.
 	ctx = featureflag.ContextWithFeatureFlag(ctx, featureflag.PraefectGeneratedReplicaPaths, rnd.Int()%2 == 0)
+	// Many tests use git commands, but few of them enable TRACE2, and most also don't need it.
+	ctx = featureflag.ContextWithFeatureFlag(ctx, featureflag.GitTrace2, false)
 
 	for _, opt := range opts {
 		ctx = opt(ctx)