Implement Git trace2 events parser

Git v2.20 introduces new Trace2 API. This API provides kinda hook to
extract the internal telemetry information of a Git process. It writes
the data to stderr or a file. The events are structured as a flat list
of consecutive events. When GIT_TRACE2_BRIEF variable is set, timing
information is omitted. That makes this format particularly hard to use
later.

In the foreseeable future, we use trace2 in two places: generating
equivalent distributed spans and expose pack object metrics. In the
future, we are looking forward to utilizing Trace2 more. As a result, we
should create an internal representation serving our purposes.
Naturally, the data can be presented as a tree.

This commit creates a parser that transform flat events to a tree data
structure.

For more information: https://git-scm.com/docs/api-trace2

7 files changed, 743 insertions, 0 deletions

diff --git a/internal/trace2/parser.go b/internal/trace2/parser.go
new file mode 100644
index 000000000..0215bb48c
--- /dev/null
+++ b/internal/trace2/parser.go
@@ -0,0 +1,222 @@
+package trace2
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"strings"
+	"time"
+
+	"gitlab.com/gitlab-org/gitaly/v15/internal/structerr"
+	"gitlab.com/gitlab-org/gitaly/v15/internal/tracing"
+)
+
+// Parser parses the events generated by Git Trace2 API to a tree data structure. The events are
+// structured as a flat list of events. They are sorted in chronological order. Each event describes
+// a certain sub operation, including some relative data and metadata. Some events, such as
+// "region_enter" or "cmd_start", indicate a new section in which the consecutive events belong to.
+// Correspondingly, exiting events, such as "region_leave" or "atexit", exits the current section.
+// Trace2 also captures the events of children processes.
+//
+// By default, all events include "time", "file", and "line" fields. Those fields increase the size
+// and processing overhead significantly. In production environment, they can be omitted by setting
+// GIT_TRACE2_BRIEF environment variable. Afterward, we can only extract the time from the initial
+// process events. The times of other events could be inferred from the time difference relative to
+// the first event ("t_abs" field) or the current section ("t_rel" field).
+//
+// Apart from the processing events, Trace2 API also exposes useful statistical information. They
+// are stored in "data" and "data_json" event under "key" and "value" fields. They are particularly
+// useful to sample and expose internal Git metrics.
+//
+// The result of the parsing process is a root Trace node of the tree. The root node is a dummy node,
+// not a part of the original events. So, it's recommended to skip this node when walking.
+//
+// For more information, please visit Trace2 API: https://git-scm.com/docs/api-trace2
+type Parser struct {
+	root    *Trace
+	pointer *Trace
+}
+
+// timeLayout defines the absolute timestamp format of trace2 event
+const timeLayout = "2006-01-02T15:04:05.000000Z"
+
+// jsonEvent is a simplified version of Trace2 event format. Each event may have different fields
+// according to event type. The full format can be found here:
+// https://git-scm.com/docs/api-trace2#_event_format
+type jsonEvent struct {
+	Name      string           `json:"event"`
+	Category  string           `json:"category"`
+	Label     string           `json:"label"`
+	Thread    string           `json:"thread"`
+	DataKey   string           `json:"key"`
+	DataValue *json.RawMessage `json:"value"`
+	Time      string           `json:"time"`
+	TimeAbs   float64          `json:"t_abs"`
+	TimeRel   float64          `json:"t_rel"`
+	Argv      []string         `json:"argv"`
+	ChildID   int              `json:"child_id"`
+	Msg       string           `json:"msg"`
+	Code      int              `json:"code"`
+}
+
+var ignoredEvents = map[string]struct{}{
+	"version":  {},
+	"cmd_name": {},
+	"exit":     {},
+}
+
+// Parse receives a reader object and returns the root trace of the tree. The parser reads line by
+// line. Each line stores an event under JSON format. It doesn't rewind the reader.
+func (p *Parser) Parse(ctx context.Context, reader io.Reader) (*Trace, error) {
+	span, _ := tracing.StartSpanIfHasParent(ctx, "trace2.Parse", nil)
+	defer span.Finish()
+
+	p.root = nil
+	p.pointer = nil
+
+	decoder := json.NewDecoder(reader)
+	for {
+		var event jsonEvent
+		err := decoder.Decode(&event)
+		if err != nil {
+			if err == io.EOF {
+				return p.root, nil
+			}
+			return nil, fmt.Errorf("decode event: %w", err)
+		}
+
+		err = p.parseEvent(&event)
+		if err != nil {
+			p.root = nil
+			return nil, structerr.NewInternal("process event: %w", err).WithMetadata("event", event)
+		}
+	}
+}
+
+func (p *Parser) parseEvent(event *jsonEvent) error {
+	if _, ok := ignoredEvents[event.Name]; ok {
+		return nil
+	}
+
+	var trace *Trace
+	t, err := p.parseEventTime(p.pointer, event)
+	if err != nil {
+		return fmt.Errorf("parse event time: %w", err)
+	}
+
+	if p.root == nil {
+		trace = &Trace{Thread: event.Thread, StartTime: t, Name: "root"}
+		p.root = trace
+		p.pointer = p.root
+	}
+
+	if p.pointer == nil {
+		// Invalid exit sequence. Just in case
+		p.pointer = p.root
+	}
+
+	// Escape commands, don't create trace
+	switch event.Name {
+	case "atexit", "child_exit":
+		p.pointer.FinishTime = t
+		p.pointer.setMetadata("code", fmt.Sprintf("%d", event.Code))
+		p.pointer = p.pointer.Parent
+		return nil
+	case "region_leave":
+		p.pointer.FinishTime = t
+		p.pointer = p.pointer.Parent
+		return nil
+	}
+
+	trace = &Trace{
+		ChildID:    p.pointer.ChildID,
+		Thread:     event.Thread,
+		StartTime:  t,
+		FinishTime: t,
+		Parent:     p.pointer,
+		Depth:      p.pointer.Depth + 1,
+	}
+	if event.Msg != "" {
+		trace.setMetadata("msg", event.Msg)
+	}
+	p.pointer.Children = append(p.pointer.Children, trace)
+
+	switch event.Name {
+	case "start":
+		trace.setName([]string{event.Name, event.Category, event.Label})
+		trace.setMetadata("argv", strings.Join(event.Argv, " "))
+	case "child_start":
+		trace.setName([]string{event.Name, event.Category, event.Label})
+		trace.setMetadata("argv", strings.Join(event.Argv, " "))
+		trace.ChildID = fmt.Sprintf("%d", event.ChildID)
+		p.pointer = trace
+	case "region_enter":
+		trace.setName([]string{event.Category, event.Label})
+		p.pointer = trace
+	case "data":
+		trace.setName([]string{event.Name, event.Category, event.Label, event.DataKey})
+		if event.DataValue != nil {
+			var data string
+			// When the event name is "data", we can unmarshal the data. This allows
+			// easy data access later
+			err := json.Unmarshal(*event.DataValue, &data)
+			if err != nil {
+				return fmt.Errorf("mismatched data value: %w", err)
+			}
+			trace.setMetadata("data", data)
+		}
+	case "data_json":
+		trace.setName([]string{event.Name, event.Category, event.Label, event.DataKey})
+		if event.DataValue != nil {
+			trace.setMetadata("data", string(*event.DataValue))
+		}
+	default:
+		trace.setName([]string{event.Name, event.Category, event.Label})
+	}
+	return nil
+}
+
+func (p *Parser) parseEventTime(parent *Trace, event *jsonEvent) (time.Time, error) {
+	// Absolute time. If GIT_TRACE2_BRIEF env variable is set this field is attached to the
+	// first event only. Other event's time must be inferred from time diff (TimeAbs and TimeRel)
+	if event.Time != "" {
+		return time.Parse(timeLayout, event.Time)
+	}
+
+	// Absolute time difference from the root
+	if event.TimeAbs != 0 {
+		return p.addTime(p.rootTime(), event.TimeAbs), nil
+	}
+
+	var parentTime time.Time
+	if parent != nil {
+		parentTime = parent.StartTime
+	} else {
+		parentTime = p.rootTime()
+	}
+
+	// Relative time difference from its parent
+	if event.TimeRel != 0 {
+		return p.addTime(parentTime, event.TimeRel), nil
+	}
+
+	// If an event doesn't have either TimeAbs and TimeRel, infer the time from its prior sibling
+	if parent != nil && len(parent.Children) != 0 {
+		return parent.Children[len(parent.Children)-1].FinishTime, nil
+	}
+
+	// If the event is the only child without any further information, use its parent time
+	return parentTime, nil
+}
+
+func (p *Parser) rootTime() time.Time {
+	if p.root != nil {
+		return p.root.StartTime
+	}
+	return time.Now()
+}
+
+func (p *Parser) addTime(t time.Time, diff float64) time.Time {
+	return t.Add(time.Duration(1000000*diff) * time.Microsecond)
+}
diff --git a/internal/trace2/parser_test.go b/internal/trace2/parser_test.go
new file mode 100644
index 000000000..a6e8c13a0
--- /dev/null
+++ b/internal/trace2/parser_test.go
@@ -0,0 +1,220 @@
+package trace2
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+	"gitlab.com/gitlab-org/gitaly/v15/internal/testhelper"
+)
+
+func TestParser_Parse(t *testing.T) {
+	t.Parallel()
+
+	cases := []struct {
+		desc          string
+		events        string
+		expectedErr   string
+		expectedTrace string
+	}{
+		{
+			desc:   "empty events",
+			events: "",
+		},
+		{
+			desc:        "invalid json format",
+			events:      "hello",
+			expectedErr: "decode event: invalid character 'h' looking for beginning of value",
+		},
+		{
+			desc: "incomplete events",
+			events: `
+{"event":"version","thread":"main","time":"2023-02-22T07:24:36.291735Z","file":"common-main.c","line":50,"evt":"3","exe":"2.39.1"}
+{"event":"start","thread":"main"}
+{""
+`,
+			expectedErr: "decode event: unexpected EOF",
+		},
+		{
+			desc:        "mismatched event data type",
+			events:      `{"event":"start","thread":"main","t_abs": "1234"}`,
+			expectedErr: "decode event: json: cannot unmarshal string into Go struct field jsonEvent.t_abs of type float64",
+		},
+		{
+			desc:        "invalid time",
+			events:      `{"event":"start","thread":"main","time": "1234"}`,
+			expectedErr: `process event: parse event time: parsing time "1234" as "2006-01-02T15:04:05.000000Z": cannot parse "" as "-"`,
+		},
+		{
+			desc:        "mismatched data value format",
+			events:      `{"event":"data","thread":"main","key": "hello", "value": 123}`,
+			expectedErr: `process event: mismatched data value: json: cannot unmarshal number into Go value of type string`,
+		},
+		{
+			desc:   "sampled git status events",
+			events: string(testhelper.MustReadFile(t, "testdata/git-status.event")),
+			expectedTrace: `
+08:10:10.668546 | 08:10:10.687877 | main | root (code="0")
+08:10:10.668546 | 08:10:10.668546 | main | .version
+08:10:10.754608 | 08:10:10.754608 | main | .start (argv="git status")
+08:10:10.754608 | 08:10:10.754608 | main | .def_repo
+08:10:10.754608 | 08:10:10.754608 | main | .cmd_name
+08:10:10.754608 | 08:10:10.755102 | main | .index:do_read_index (msg=".git/index")
+08:10:10.754608 | 08:10:10.754732 | main | ..cache_tree:read
+08:10:10.758297 | 08:10:10.758297 | main | ..data:index:read/version (data="2")
+08:10:10.758318 | 08:10:10.758318 | main | ..data:index:read/cache_nr (data="1585")
+08:10:10.755102 | 08:10:10.759927 | main | .progress:Refresh index
+08:10:10.755102 | 08:10:10.759845 | main | ..index:preload
+08:10:10.759845 | 08:10:10.759862 | main | ..index:refresh
+08:10:10.763258 | 08:10:10.763258 | main | ..data:progress:total_objects (data="1585")
+08:10:10.759927 | 08:10:10.759999 | main | .status:worktrees
+08:10:10.759927 | 08:10:10.759932 | main | ..diff:setup
+08:10:10.759932 | 08:10:10.759936 | main | ..diff:write back to queue
+08:10:10.759999 | 08:10:10.761522 | main | .status:index
+08:10:10.759999 | 08:10:10.760078 | main | ..unpack_trees:unpack_trees
+08:10:10.760078 | 08:10:10.760081 | main | ..diff:setup
+08:10:10.760081 | 08:10:10.760084 | main | ..diff:write back to queue
+08:10:10.761522 | 08:10:10.769788 | main | .status:untracked
+08:10:10.761522 | 08:10:10.769753 | main | ..dir:read_directory
+08:10:10.773512 | 08:10:10.773512 | main | .data:status:count/changed (data="0")
+08:10:10.773518 | 08:10:10.773518 | main | .data:status:count/untracked (data="1")
+08:10:10.773522 | 08:10:10.773522 | main | .data:status:count/ignored (data="0")
+08:10:10.773522 | 08:10:10.773727 | main | .status:print
+08:10:10.773741 | 08:10:10.773741 | main | .exit
+08:10:10.773750 | 08:10:10.773750 | main | .data_json:traverse_trees:statistics (data="{\"traverse_trees_count\":1,\"traverse_trees_max_depth\":1}")
+`,
+		},
+		{
+			desc:   "sampled git fetch events",
+			events: string(testhelper.MustReadFile(t, "testdata/git-fetch.event")),
+			expectedTrace: `
+07:24:36.291735 | 07:24:40.554407 | main | root (code="0")
+07:24:36.291735 | 07:24:36.291735 | main | .version
+07:24:36.293932 | 07:24:36.293932 | main | .start (argv="git fetch origin master")
+07:24:36.293932 | 07:24:36.293932 | main | .def_repo
+07:24:36.293932 | 07:24:36.293932 | main | .cmd_name
+07:24:36.293932 | 07:24:36.294119 | main | .index:do_read_index (msg=".git/index")
+07:24:36.293932 | 07:24:36.293975 | main | ..cache_tree:read
+07:24:36.294718 | 07:24:36.294718 | main | ..data:index:read/version (data="2")
+07:24:36.294725 | 07:24:36.294725 | main | ..data:index:read/cache_nr (data="1589")
+07:24:36.294119 | 07:24:36.330739 | main | .fetch:remote_refs (code="0")
+07:24:36.294119 | 07:24:40.128447 | main (child 0) | ..child_start (argv="ssh -o SendEnv=GIT_PROTOCOL git@gitlab.com git-upload-pack 'gitlab-org/gitaly.git'")
+07:24:39.314775 | 07:24:39.314775 | main (child 0) | ...data:transfer:negotiated-version (data="2")
+07:24:40.128447 | 07:24:36.324327 | main (child 1) | ..child_start (argv="git rev-list --objects --stdin --not --all --quiet --alternate-refs" code="0")
+07:24:40.148998 | 07:24:40.148998 | main (child 1) | ...version
+07:24:36.295636 | 07:24:36.295636 | main (child 1) | ...start (argv="git rev-list --objects --stdin --not --all --quiet --alternate-refs")
+07:24:36.295636 | 07:24:36.295636 | main (child 1) | ...def_repo
+07:24:36.295636 | 07:24:36.295636 | main (child 1) | ...cmd_name
+07:24:36.324300 | 07:24:36.324300 | main (child 1) | ...exit
+07:24:36.330739 | 07:24:36.331472 | main | .fetch:consume_refs
+07:24:36.291735 | 07:24:36.291820 | main | .submodule:parallel/fetch (msg="max:1")
+07:24:36.291820 | 07:24:36.297895 | main (child 2) | .child_start (argv="git maintenance run --auto --no-quiet" code="0")
+07:24:40.548630 | 07:24:40.548630 | main (child 2) | ..version
+07:24:36.294627 | 07:24:36.294627 | main (child 2) | ..start (argv="git maintenance run --auto --no-quiet")
+07:24:36.294627 | 07:24:36.294627 | main (child 2) | ..def_repo
+07:24:36.294627 | 07:24:36.294627 | main (child 2) | ..cmd_name
+07:24:36.297878 | 07:24:36.297878 | main (child 2) | ..exit
+07:24:40.554393 | 07:24:40.554393 | main | .exit
+`,
+		},
+		{
+			desc:   "sampled git commit events",
+			events: string(testhelper.MustReadFile(t, "testdata/git-commit.event")),
+			expectedTrace: `
+11:26:37.174893 | 11:26:38.677971 | main | root (code="0")
+11:26:37.174893 | 11:26:37.174893 | main | .version
+11:26:37.180753 | 11:26:37.180753 | main | .start (argv="git commit --amend")
+11:26:37.180753 | 11:26:37.180753 | main | .def_repo
+11:26:37.180753 | 11:26:37.180753 | main | .cmd_name
+11:26:37.180753 | 11:26:37.181072 | main | .index:do_read_index (msg=".git/index")
+11:26:37.180753 | 11:26:37.180832 | main | ..cache_tree:read
+11:26:37.185091 | 11:26:37.185091 | main | ..data:index:read/version (data="2")
+11:26:37.185105 | 11:26:37.185105 | main | ..data:index:read/cache_nr (data="1590")
+11:26:37.181072 | 11:26:37.187270 | main | .index:preload
+11:26:37.191313 | 11:26:37.191313 | main | ..data:index:preload/sum_lstat (data="1590")
+11:26:37.187270 | 11:26:37.187432 | main | .index:preload
+11:26:37.191812 | 11:26:37.191812 | main | ..data:index:preload/sum_lstat (data="0")
+11:26:37.187432 | 11:26:37.187492 | main | .index:refresh
+11:26:37.191874 | 11:26:37.191874 | main | ..data:index:refresh/sum_lstat (data="0")
+11:26:37.191906 | 11:26:37.191906 | main | ..data:index:refresh/sum_scan (data="0")
+11:26:37.187492 | 11:26:37.194463 | main | .cache_tree:update
+11:26:37.194463 | 11:26:37.194919 | main | .index:do_write_index (msg="/gitaly/.git/index.lock")
+11:26:37.194463 | 11:26:37.194533 | main | ..cache_tree:write
+11:26:37.199369 | 11:26:37.199369 | main | ..data:index:write/version (data="2")
+11:26:37.199382 | 11:26:37.199382 | main | ..data:index:write/cache_nr (data="1590")
+11:26:37.194919 | 11:26:37.194991 | main | .status:worktrees
+11:26:37.194919 | 11:26:37.194927 | main | ..diff:setup
+11:26:37.194927 | 11:26:37.194933 | main | ..diff:write back to queue
+11:26:37.194991 | 11:26:37.195415 | main | .status:index
+11:26:37.194991 | 11:26:37.195147 | main | ..unpack_trees:unpack_trees
+11:26:37.195147 | 11:26:37.195154 | main | ..diff:setup
+11:26:37.195154 | 11:26:37.195162 | main | ..diff:write back to queue
+11:26:37.195415 | 11:26:37.208155 | main | .status:untracked
+11:26:37.195415 | 11:26:37.208106 | main | ..dir:read_directory
+11:26:37.214048 | 11:26:37.214048 | main | .data:status:count/changed (data="5")
+11:26:37.214055 | 11:26:37.214055 | main | .data:status:count/untracked (data="0")
+11:26:37.214060 | 11:26:37.214060 | main | .data:status:count/ignored (data="0")
+11:26:37.214060 | 11:26:37.214271 | main | .status:print
+11:26:37.214271 | 11:26:37.214297 | main | .cache_tree:update
+11:26:37.214297 | 11:26:37.176472 | main (child 0) | .child_start (argv="nvim /gitaly/.git/COMMIT_EDITMSG" code="128")
+11:26:37.353241 | 11:26:37.353241 | main (child 0) | ..version
+11:26:37.175671 | 11:26:37.175671 | main (child 0) | ..start (argv="git diff --no-color --no-ext-diff -U0 -- COMMIT_EDITMSG")
+11:26:37.175671 | 11:26:37.175671 | main (child 0) | ..cmd_name
+11:26:37.175671 | 11:26:37.175671 | main (child 0) | ..error (msg="this operation must be run in a work tree")
+11:26:37.176465 | 11:26:37.176465 | main (child 0) | ..exit
+11:26:38.624567 | 11:26:38.624567 | main | .version
+11:26:37.175517 | 11:26:37.175517 | main | .start (argv="git branch --no-color --show-current")
+11:26:37.175517 | 11:26:37.175517 | main | .def_repo
+11:26:37.175517 | 11:26:37.175517 | main | .cmd_name
+11:26:37.176095 | 11:26:37.176095 | main | .exit
+11:26:37.174893 | 11:26:37.180102 | main (child 1) | .child_start (argv="git maintenance run --auto --no-quiet" code="0")
+11:26:38.667832 | 11:26:38.667832 | main (child 1) | ..version
+11:26:37.178802 | 11:26:37.178802 | main (child 1) | ..start (argv="git maintenance run --auto --no-quiet")
+11:26:37.178802 | 11:26:37.178802 | main (child 1) | ..def_repo
+11:26:37.178802 | 11:26:37.178802 | main (child 1) | ..cmd_name
+11:26:37.180092 | 11:26:37.180092 | main (child 1) | ..exit
+11:26:37.174893 | 11:26:37.174907 | main | .diff:setup
+11:26:37.174907 | 11:26:37.174912 | main | .diff:write back to queue
+11:26:38.677953 | 11:26:38.677953 | main | .exit
+11:26:38.677965 | 11:26:38.677965 | main | .data_json:traverse_trees:statistics (data="{\"traverse_trees_count\":2,\"traverse_trees_max_depth\":2}")
+`,
+		},
+		{
+			desc:   "sampled git pack objects events",
+			events: string(testhelper.MustReadFile(t, "testdata/git-pack-objects.event")),
+			expectedTrace: `
+12:05:04.840009 | 12:05:04.848504 | main | root (code="0")
+12:05:04.840009 | 12:05:04.840009 | main | .version
+12:05:04.842347 | 12:05:04.842347 | main | .start (argv="git pack-objects toon --compression=0")
+12:05:04.842347 | 12:05:04.842347 | main | .def_repo
+12:05:04.842347 | 12:05:04.842347 | main | .cmd_name
+12:05:04.842347 | 12:05:04.843782 | main | .pack-objects:enumerate-objects
+12:05:04.842347 | 12:05:04.843430 | main | ..progress:Enumerating objects
+12:05:04.843782 | 12:05:04.843872 | main | .pack-objects:prepare-pack
+12:05:04.843782 | 12:05:04.843857 | main | ..progress:Counting objects
+12:05:04.843872 | 12:05:04.847874 | main | .pack-objects:write-pack-file
+12:05:04.843872 | 12:05:04.847844 | main | ..progress:Writing objects
+12:05:04.848460 | 12:05:04.848460 | main | ..data:pack-objects:write_pack_file/wrote (data="1")
+12:05:04.848491 | 12:05:04.848491 | main | .data:fsync:fsync/writeout-only (data="2")
+12:05:04.848498 | 12:05:04.848498 | main | .exit
+`,
+		},
+	}
+	for _, tc := range cases {
+		tc := tc
+		t.Run(tc.desc, func(t *testing.T) {
+			parser := &Parser{}
+			trace, err := parser.Parse(testhelper.Context(t), strings.NewReader(tc.events))
+
+			if tc.expectedErr != "" {
+				// JSON doesn't export error creation. It's not feasible to compare
+				// error directly. We can only compare by the stringified error
+				require.Error(t, err)
+				require.Equal(t, tc.expectedErr, err.Error())
+			} else {
+				require.NoError(t, err)
+			}
+			require.Equal(t, strings.TrimSpace(tc.expectedTrace), trace.Inspect())
+		})
+	}
+}
diff --git a/internal/trace2/testdata/git-commit.event b/internal/trace2/testdata/git-commit.event
new file mode 100644
index 000000000..70f5e5748
--- /dev/null
+++ b/internal/trace2/testdata/git-commit.event
@@ -0,0 +1,82 @@
+{"event":"version","sid":"1234/20230222T112637.174491Z-Ha0f0bee5-P0000b286","thread":"main","time":"2023-02-22T11:26:37.174893Z","file":"common-main.c","line":50,"evt":"3","exe":"2.39.1"}
+{"event":"start","sid":"1234/20230222T112637.174491Z-Ha0f0bee5-P0000b286","thread":"main","file":"common-main.c","line":51,"t_abs":0.005860,"argv":["git","commit","--amend"]}
+{"event":"def_repo","sid":"1234/20230222T112637.174491Z-Ha0f0bee5-P0000b286","thread":"main","file":"repository.c","line":136,"repo":1,"worktree":"/gitaly"}
+{"event":"cmd_name","sid":"1234/20230222T112637.174491Z-Ha0f0bee5-P0000b286","thread":"main","file":"git.c","line":461,"name":"commit","hierarchy":"commit"}
+{"event":"region_enter","sid":"1234/20230222T112637.174491Z-Ha0f0bee5-P0000b286","thread":"main","file":"read-cache.c","line":2478,"repo":1,"nesting":1,"category":"index","label":"do_read_index","msg":".git/index"}
+{"event":"region_enter","sid":"1234/20230222T112637.174491Z-Ha0f0bee5-P0000b286","thread":"main","file":"cache-tree.c","line":628,"repo":1,"nesting":2,"category":"cache_tree","label":"read"}
+{"event":"region_leave","sid":"1234/20230222T112637.174491Z-Ha0f0bee5-P0000b286","thread":"main","file":"cache-tree.c","line":630,"repo":1,"t_rel":0.000079,"nesting":2,"category":"cache_tree","label":"read"}
+{"event":"data","sid":"1234/20230222T112637.174491Z-Ha0f0bee5-P0000b286","thread":"main","file":"read-cache.c","line":2424,"repo":1,"t_abs":0.010198,"t_rel":0.000289,"nesting":2,"category":"index","key":"read/version","value":"2"}
+{"event":"data","sid":"1234/20230222T112637.174491Z-Ha0f0bee5-P0000b286","thread":"main","file":"read-cache.c","line":2426,"repo":1,"t_abs":0.010212,"t_rel":0.000303,"nesting":2,"category":"index","key":"read/cache_nr","value":"1590"}
+{"event":"region_leave","sid":"1234/20230222T112637.174491Z-Ha0f0bee5-P0000b286","thread":"main","file":"read-cache.c","line":2483,"repo":1,"t_rel":0.000319,"nesting":1,"category":"index","label":"do_read_index","msg":".git/index"}
+{"event":"region_enter","sid":"1234/20230222T112637.174491Z-Ha0f0bee5-P0000b286","thread":"main","file":"preload-index.c","line":114,"nesting":1,"category":"index","label":"preload"}
+{"event":"data","sid":"1234/20230222T112637.174491Z-Ha0f0bee5-P0000b286","thread":"main","file":"preload-index.c","line":162,"t_abs":0.016420,"t_rel":0.006176,"nesting":2,"category":"index","key":"preload/sum_lstat","value":"1590"}
+{"event":"region_leave","sid":"1234/20230222T112637.174491Z-Ha0f0bee5-P0000b286","thread":"main","file":"preload-index.c","line":163,"t_rel":0.006198,"nesting":1,"category":"index","label":"preload"}
+{"event":"region_enter","sid":"1234/20230222T112637.174491Z-Ha0f0bee5-P0000b286","thread":"main","file":"preload-index.c","line":114,"nesting":1,"category":"index","label":"preload"}
+{"event":"data","sid":"1234/20230222T112637.174491Z-Ha0f0bee5-P0000b286","thread":"main","file":"preload-index.c","line":162,"t_abs":0.016919,"t_rel":0.000134,"nesting":2,"category":"index","key":"preload/sum_lstat","value":"0"}
+{"event":"region_leave","sid":"1234/20230222T112637.174491Z-Ha0f0bee5-P0000b286","thread":"main","file":"preload-index.c","line":163,"t_rel":0.000162,"nesting":1,"category":"index","label":"preload"}
+{"event":"region_enter","sid":"1234/20230222T112637.174491Z-Ha0f0bee5-P0000b286","thread":"main","file":"read-cache.c","line":1639,"nesting":1,"category":"index","label":"refresh"}
+{"event":"data","sid":"1234/20230222T112637.174491Z-Ha0f0bee5-P0000b286","thread":"main","file":"read-cache.c","line":1721,"t_abs":0.016981,"t_rel":0.000019,"nesting":2,"category":"index","key":"refresh/sum_lstat","value":"0"}
+{"event":"data","sid":"1234/20230222T112637.174491Z-Ha0f0bee5-P0000b286","thread":"main","file":"read-cache.c","line":1722,"t_abs":0.017013,"t_rel":0.000051,"nesting":2,"category":"index","key":"refresh/sum_scan","value":"0"}
+{"event":"region_leave","sid":"1234/20230222T112637.174491Z-Ha0f0bee5-P0000b286","thread":"main","file":"read-cache.c","line":1723,"t_rel":0.000060,"nesting":1,"category":"index","label":"refresh"}
+{"event":"region_enter","sid":"1234/20230222T112637.174491Z-Ha0f0bee5-P0000b286","thread":"main","file":"cache-tree.c","line":477,"repo":1,"nesting":1,"category":"cache_tree","label":"update"}
+{"event":"region_leave","sid":"1234/20230222T112637.174491Z-Ha0f0bee5-P0000b286","thread":"main","file":"cache-tree.c","line":482,"repo":1,"t_rel":0.006971,"nesting":1,"category":"cache_tree","label":"update"}
+{"event":"region_enter","sid":"1234/20230222T112637.174491Z-Ha0f0bee5-P0000b286","thread":"main","file":"read-cache.c","line":3208,"repo":1,"nesting":1,"category":"index","label":"do_write_index","msg":"/gitaly/.git/index.lock"}
+{"event":"region_enter","sid":"1234/20230222T112637.174491Z-Ha0f0bee5-P0000b286","thread":"main","file":"cache-tree.c","line":532,"repo":1,"nesting":2,"category":"cache_tree","label":"write"}
+{"event":"region_leave","sid":"1234/20230222T112637.174491Z-Ha0f0bee5-P0000b286","thread":"main","file":"cache-tree.c","line":534,"repo":1,"t_rel":0.000070,"nesting":2,"category":"cache_tree","label":"write"}
+{"event":"data","sid":"1234/20230222T112637.174491Z-Ha0f0bee5-P0000b286","thread":"main","file":"read-cache.c","line":3170,"repo":1,"t_abs":0.024476,"t_rel":0.000432,"nesting":2,"category":"index","key":"write/version","value":"2"}
+{"event":"data","sid":"1234/20230222T112637.174491Z-Ha0f0bee5-P0000b286","thread":"main","file":"read-cache.c","line":3172,"repo":1,"t_abs":0.024489,"t_rel":0.000445,"nesting":2,"category":"index","key":"write/cache_nr","value":"1590"}
+{"event":"region_leave","sid":"1234/20230222T112637.174491Z-Ha0f0bee5-P0000b286","thread":"main","file":"read-cache.c","line":3211,"repo":1,"t_rel":0.000456,"nesting":1,"category":"index","label":"do_write_index","msg":"/gitaly/.git/index.lock"}
+{"event":"region_enter","sid":"1234/20230222T112637.174491Z-Ha0f0bee5-P0000b286","thread":"main","file":"wt-status.c","line":817,"repo":1,"nesting":1,"category":"status","label":"worktrees"}
+{"event":"region_enter","sid":"1234/20230222T112637.174491Z-Ha0f0bee5-P0000b286","thread":"main","file":"diffcore-rename.c","line":1401,"repo":1,"nesting":2,"category":"diff","label":"setup"}
+{"event":"region_leave","sid":"1234/20230222T112637.174491Z-Ha0f0bee5-P0000b286","thread":"main","file":"diffcore-rename.c","line":1454,"repo":1,"t_rel":0.000008,"nesting":2,"category":"diff","label":"setup"}
+{"event":"region_enter","sid":"1234/20230222T112637.174491Z-Ha0f0bee5-P0000b286","thread":"main","file":"diffcore-rename.c","line":1633,"repo":1,"nesting":2,"category":"diff","label":"write back to queue"}
+{"event":"region_leave","sid":"1234/20230222T112637.174491Z-Ha0f0bee5-P0000b286","thread":"main","file":"diffcore-rename.c","line":1710,"repo":1,"t_rel":0.000006,"nesting":2,"category":"diff","label":"write back to queue"}
+{"event":"region_leave","sid":"1234/20230222T112637.174491Z-Ha0f0bee5-P0000b286","thread":"main","file":"wt-status.c","line":819,"repo":1,"t_rel":0.000072,"nesting":1,"category":"status","label":"worktrees"}
+{"event":"region_enter","sid":"1234/20230222T112637.174491Z-Ha0f0bee5-P0000b286","thread":"main","file":"wt-status.c","line":826,"repo":1,"nesting":1,"category":"status","label":"index"}
+{"event":"region_enter","sid":"1234/20230222T112637.174491Z-Ha0f0bee5-P0000b286","thread":"main","file":"unpack-trees.c","line":1875,"repo":1,"nesting":2,"category":"unpack_trees","label":"unpack_trees"}
+{"event":"region_leave","sid":"1234/20230222T112637.174491Z-Ha0f0bee5-P0000b286","thread":"main","file":"unpack-trees.c","line":2068,"repo":1,"t_rel":0.000156,"nesting":2,"category":"unpack_trees","label":"unpack_trees"}
+{"event":"region_enter","sid":"1234/20230222T112637.174491Z-Ha0f0bee5-P0000b286","thread":"main","file":"diffcore-rename.c","line":1401,"repo":1,"nesting":2,"category":"diff","label":"setup"}
+{"event":"region_leave","sid":"1234/20230222T112637.174491Z-Ha0f0bee5-P0000b286","thread":"main","file":"diffcore-rename.c","line":1454,"repo":1,"t_rel":0.000007,"nesting":2,"category":"diff","label":"setup"}
+{"event":"region_enter","sid":"1234/20230222T112637.174491Z-Ha0f0bee5-P0000b286","thread":"main","file":"diffcore-rename.c","line":1633,"repo":1,"nesting":2,"category":"diff","label":"write back to queue"}
+{"event":"region_leave","sid":"1234/20230222T112637.174491Z-Ha0f0bee5-P0000b286","thread":"main","file":"diffcore-rename.c","line":1710,"repo":1,"t_rel":0.000008,"nesting":2,"category":"diff","label":"write back to queue"}
+{"event":"region_leave","sid":"1234/20230222T112637.174491Z-Ha0f0bee5-P0000b286","thread":"main","file":"wt-status.c","line":828,"repo":1,"t_rel":0.000424,"nesting":1,"category":"status","label":"index"}
+{"event":"region_enter","sid":"1234/20230222T112637.174491Z-Ha0f0bee5-P0000b286","thread":"main","file":"wt-status.c","line":831,"repo":1,"nesting":1,"category":"status","label":"untracked"}
+{"event":"region_enter","sid":"1234/20230222T112637.174491Z-Ha0f0bee5-P0000b286","thread":"main","file":"dir.c","line":2988,"repo":1,"nesting":2,"category":"dir","label":"read_directory"}
+{"event":"region_leave","sid":"1234/20230222T112637.174491Z-Ha0f0bee5-P0000b286","thread":"main","file":"dir.c","line":3011,"repo":1,"t_rel":0.012691,"nesting":2,"category":"dir","label":"read_directory"}
+{"event":"region_leave","sid":"1234/20230222T112637.174491Z-Ha0f0bee5-P0000b286","thread":"main","file":"wt-status.c","line":833,"repo":1,"t_rel":0.012740,"nesting":1,"category":"status","label":"untracked"}
+{"event":"data","sid":"1234/20230222T112637.174491Z-Ha0f0bee5-P0000b286","thread":"main","file":"wt-status.c","line":2507,"repo":1,"t_abs":0.039155,"t_rel":0.039155,"nesting":1,"category":"status","key":"count/changed","value":"5"}
+{"event":"data","sid":"1234/20230222T112637.174491Z-Ha0f0bee5-P0000b286","thread":"main","file":"wt-status.c","line":2509,"repo":1,"t_abs":0.039162,"t_rel":0.039162,"nesting":1,"category":"status","key":"count/untracked","value":"0"}
+{"event":"data","sid":"1234/20230222T112637.174491Z-Ha0f0bee5-P0000b286","thread":"main","file":"wt-status.c","line":2510,"repo":1,"t_abs":0.039167,"t_rel":0.039167,"nesting":1,"category":"status","key":"count/ignored","value":"0"}
+{"event":"region_enter","sid":"1234/20230222T112637.174491Z-Ha0f0bee5-P0000b286","thread":"main","file":"wt-status.c","line":2512,"repo":1,"nesting":1,"category":"status","label":"print"}
+{"event":"region_leave","sid":"1234/20230222T112637.174491Z-Ha0f0bee5-P0000b286","thread":"main","file":"wt-status.c","line":2533,"repo":1,"t_rel":0.000211,"nesting":1,"category":"status","label":"print"}
+{"event":"region_enter","sid":"1234/20230222T112637.174491Z-Ha0f0bee5-P0000b286","thread":"main","file":"cache-tree.c","line":477,"repo":1,"nesting":1,"category":"cache_tree","label":"update"}
+{"event":"region_leave","sid":"1234/20230222T112637.174491Z-Ha0f0bee5-P0000b286","thread":"main","file":"cache-tree.c","line":482,"repo":1,"t_rel":0.000026,"nesting":1,"category":"cache_tree","label":"update"}
+{"event":"child_start","sid":"1234/20230222T112637.174491Z-Ha0f0bee5-P0000b286","thread":"main","file":"run-command.c","line":722,"child_id":0,"child_class":"editor","use_shell":true,"argv":["nvim","/gitaly/.git/COMMIT_EDITMSG"]}
+{"event":"version","sid":"1234/20230222T112637.174491Z-Ha0f0bee5-P0000b286/20230222T112637.352885Z-Ha0f0bee5-P0000b289","thread":"main","time":"2023-02-22T11:26:37.353241Z","file":"common-main.c","line":50,"evt":"3","exe":"2.39.1"}
+{"event":"start","sid":"1234/20230222T112637.174491Z-Ha0f0bee5-P0000b286/20230222T112637.352885Z-Ha0f0bee5-P0000b289","thread":"main","file":"common-main.c","line":51,"t_abs":0.000778,"argv":["git","diff","--no-color","--no-ext-diff","-U0","--","COMMIT_EDITMSG"]}
+{"event":"cmd_name","sid":"1234/20230222T112637.174491Z-Ha0f0bee5-P0000b286/20230222T112637.352885Z-Ha0f0bee5-P0000b289","thread":"main","file":"git.c","line":461,"name":"diff","hierarchy":"commit/diff"}
+{"event":"error","sid":"1234/20230222T112637.174491Z-Ha0f0bee5-P0000b286/20230222T112637.352885Z-Ha0f0bee5-P0000b289","thread":"main","file":"usage.c","line":60,"msg":"this operation must be run in a work tree","fmt":"this operation must be run in a work tree"}
+{"event":"exit","sid":"1234/20230222T112637.174491Z-Ha0f0bee5-P0000b286/20230222T112637.352885Z-Ha0f0bee5-P0000b289","thread":"main","file":"usage.c","line":74,"t_abs":0.001572,"code":128}
+{"event":"atexit","sid":"1234/20230222T112637.174491Z-Ha0f0bee5-P0000b286/20230222T112637.352885Z-Ha0f0bee5-P0000b289","thread":"main","file":"trace2/tr2_tgt_event.c","line":204,"t_abs":0.001579,"code":128}
+{"event":"version","sid":"1234/20230222T112637.174491Z-Ha0f0bee5-P0000b286/20230222T112638.624303Z-Ha0f0bee5-P0000b28e","thread":"main","time":"2023-02-22T11:26:38.624567Z","file":"common-main.c","line":50,"evt":"3","exe":"2.39.1"}
+{"event":"start","sid":"1234/20230222T112637.174491Z-Ha0f0bee5-P0000b286/20230222T112638.624303Z-Ha0f0bee5-P0000b28e","thread":"main","file":"common-main.c","line":51,"t_abs":0.000624,"argv":["git","branch","--no-color","--show-current"]}
+{"event":"def_repo","sid":"1234/20230222T112637.174491Z-Ha0f0bee5-P0000b286/20230222T112638.624303Z-Ha0f0bee5-P0000b28e","thread":"main","file":"repository.c","line":136,"repo":1,"worktree":"/gitaly"}
+{"event":"cmd_name","sid":"1234/20230222T112637.174491Z-Ha0f0bee5-P0000b286/20230222T112638.624303Z-Ha0f0bee5-P0000b28e","thread":"main","file":"git.c","line":461,"name":"branch","hierarchy":"commit/branch"}
+{"event":"exit","sid":"1234/20230222T112637.174491Z-Ha0f0bee5-P0000b286/20230222T112638.624303Z-Ha0f0bee5-P0000b28e","thread":"main","file":"git.c","line":721,"t_abs":0.001202,"code":0}
+{"event":"atexit","sid":"1234/20230222T112637.174491Z-Ha0f0bee5-P0000b286/20230222T112638.624303Z-Ha0f0bee5-P0000b28e","thread":"main","file":"trace2/tr2_tgt_event.c","line":204,"t_abs":0.001209,"code":0}
+{"event":"child_exit","sid":"1234/20230222T112637.174491Z-Ha0f0bee5-P0000b286","thread":"main","file":"run-command.c","line":979,"child_id":0,"pid":45704,"code":0,"t_rel":1.449656}
+{"event":"child_start","sid":"1234/20230222T112637.174491Z-Ha0f0bee5-P0000b286","thread":"main","file":"run-command.c","line":722,"child_id":1,"child_class":"?","use_shell":false,"argv":["git","maintenance","run","--auto","--no-quiet"]}
+{"event":"version","sid":"1234/20230222T112637.174491Z-Ha0f0bee5-P0000b286/20230222T112638.667670Z-Ha0f0bee5-P0000b292","thread":"main","time":"2023-02-22T11:26:38.667832Z","file":"common-main.c","line":50,"evt":"3","exe":"2.39.1"}
+{"event":"start","sid":"1234/20230222T112637.174491Z-Ha0f0bee5-P0000b286/20230222T112638.667670Z-Ha0f0bee5-P0000b292","thread":"main","file":"common-main.c","line":51,"t_abs":0.003909,"argv":["git","maintenance","run","--auto","--no-quiet"]}
+{"event":"def_repo","sid":"1234/20230222T112637.174491Z-Ha0f0bee5-P0000b286/20230222T112638.667670Z-Ha0f0bee5-P0000b292","thread":"main","file":"repository.c","line":136,"repo":1,"worktree":"/gitaly"}
+{"event":"cmd_name","sid":"1234/20230222T112637.174491Z-Ha0f0bee5-P0000b286/20230222T112638.667670Z-Ha0f0bee5-P0000b292","thread":"main","file":"git.c","line":461,"name":"maintenance","hierarchy":"commit/maintenance"}
+{"event":"exit","sid":"1234/20230222T112637.174491Z-Ha0f0bee5-P0000b286/20230222T112638.667670Z-Ha0f0bee5-P0000b292","thread":"main","file":"git.c","line":721,"t_abs":0.005199,"code":0}
+{"event":"atexit","sid":"1234/20230222T112637.174491Z-Ha0f0bee5-P0000b286/20230222T112638.667670Z-Ha0f0bee5-P0000b292","thread":"main","file":"trace2/tr2_tgt_event.c","line":204,"t_abs":0.005209,"code":0}
+{"event":"child_exit","sid":"1234/20230222T112637.174491Z-Ha0f0bee5-P0000b286","thread":"main","file":"run-command.c","line":979,"child_id":1,"pid":45714,"code":0,"t_rel":0.008935}
+{"event":"region_enter","sid":"1234/20230222T112637.174491Z-Ha0f0bee5-P0000b286","thread":"main","file":"diffcore-rename.c","line":1401,"repo":1,"nesting":1,"category":"diff","label":"setup"}
+{"event":"region_leave","sid":"1234/20230222T112637.174491Z-Ha0f0bee5-P0000b286","thread":"main","file":"diffcore-rename.c","line":1454,"repo":1,"t_rel":0.000014,"nesting":1,"category":"diff","label":"setup"}
+{"event":"region_enter","sid":"1234/20230222T112637.174491Z-Ha0f0bee5-P0000b286","thread":"main","file":"diffcore-rename.c","line":1633,"repo":1,"nesting":1,"category":"diff","label":"write back to queue"}
+{"event":"region_leave","sid":"1234/20230222T112637.174491Z-Ha0f0bee5-P0000b286","thread":"main","file":"diffcore-rename.c","line":1710,"repo":1,"t_rel":0.000005,"nesting":1,"category":"diff","label":"write back to queue"}
+{"event":"exit","sid":"1234/20230222T112637.174491Z-Ha0f0bee5-P0000b286","thread":"main","file":"git.c","line":721,"t_abs":1.503060,"code":0}
+{"event":"data_json","sid":"1234/20230222T112637.174491Z-Ha0f0bee5-P0000b286","thread":"main","file":"tree-walk.c","line":189,"repo":1,"t_abs":1.503072,"t_rel":1.503072,"nesting":1,"category":"traverse_trees","key":"statistics","value":{"traverse_trees_count":2,"traverse_trees_max_depth":2}}
+{"event":"atexit","sid":"1234/20230222T112637.174491Z-Ha0f0bee5-P0000b286","thread":"main","file":"trace2/tr2_tgt_event.c","line":204,"t_abs":1.503078,"code":0}
diff --git a/internal/trace2/testdata/git-fetch.event b/internal/trace2/testdata/git-fetch.event
new file mode 100644
index 000000000..7d68ee352
--- /dev/null
+++ b/internal/trace2/testdata/git-fetch.event
@@ -0,0 +1,37 @@
+{"event":"version","sid":"1234/20230222T072436.291562Z-Ha0f0bee5-P00007515","thread":"main","time":"2023-02-22T07:24:36.291735Z","file":"common-main.c","line":50,"evt":"3","exe":"2.39.1"}
+{"event":"start","sid":"1234/20230222T072436.291562Z-Ha0f0bee5-P00007515","thread":"main","file":"common-main.c","line":51,"t_abs":0.002197,"argv":["git","fetch","origin","master"]}
+{"event":"def_repo","sid":"1234/20230222T072436.291562Z-Ha0f0bee5-P00007515","thread":"main","file":"repository.c","line":136,"repo":1,"worktree":"/gitaly"}
+{"event":"cmd_name","sid":"1234/20230222T072436.291562Z-Ha0f0bee5-P00007515","thread":"main","file":"git.c","line":461,"name":"fetch","hierarchy":"fetch"}
+{"event":"region_enter","sid":"1234/20230222T072436.291562Z-Ha0f0bee5-P00007515","thread":"main","file":"read-cache.c","line":2478,"repo":1,"nesting":1,"category":"index","label":"do_read_index","msg":".git/index"}
+{"event":"region_enter","sid":"1234/20230222T072436.291562Z-Ha0f0bee5-P00007515","thread":"main","file":"cache-tree.c","line":628,"repo":1,"nesting":2,"category":"cache_tree","label":"read"}
+{"event":"region_leave","sid":"1234/20230222T072436.291562Z-Ha0f0bee5-P00007515","thread":"main","file":"cache-tree.c","line":630,"repo":1,"t_rel":0.000043,"nesting":2,"category":"cache_tree","label":"read"}
+{"event":"data","sid":"1234/20230222T072436.291562Z-Ha0f0bee5-P00007515","thread":"main","file":"read-cache.c","line":2424,"repo":1,"t_abs":0.002983,"t_rel":0.000174,"nesting":2,"category":"index","key":"read/version","value":"2"}
+{"event":"data","sid":"1234/20230222T072436.291562Z-Ha0f0bee5-P00007515","thread":"main","file":"read-cache.c","line":2426,"repo":1,"t_abs":0.002990,"t_rel":0.000181,"nesting":2,"category":"index","key":"read/cache_nr","value":"1589"}
+{"event":"region_leave","sid":"1234/20230222T072436.291562Z-Ha0f0bee5-P00007515","thread":"main","file":"read-cache.c","line":2483,"repo":1,"t_rel":0.000187,"nesting":1,"category":"index","label":"do_read_index","msg":".git/index"}
+{"event":"region_enter","sid":"1234/20230222T072436.291562Z-Ha0f0bee5-P00007515","thread":"main","file":"builtin/fetch.c","line":1646,"repo":1,"nesting":1,"category":"fetch","label":"remote_refs"}
+{"event":"child_start","sid":"1234/20230222T072436.291562Z-Ha0f0bee5-P00007515","thread":"main","file":"run-command.c","line":722,"child_id":0,"child_class":"transport/ssh","use_shell":false,"argv":["ssh","-o","SendEnv=GIT_PROTOCOL","git@gitlab.com","git-upload-pack 'gitlab-org/gitaly.git'"]}
+{"event":"data","sid":"1234/20230222T072436.291562Z-Ha0f0bee5-P00007515","thread":"main","file":"connect.c","line":167,"t_abs":3.023040,"t_rel":3.018289,"nesting":2,"category":"transfer","key":"negotiated-version","value":"2"}
+{"event":"region_leave","sid":"1234/20230222T072436.291562Z-Ha0f0bee5-P00007515","thread":"main","file":"builtin/fetch.c","line":1649,"repo":1,"t_rel":3.834328,"nesting":1,"category":"fetch","label":"remote_refs"}
+{"event":"child_start","sid":"1234/20230222T072436.291562Z-Ha0f0bee5-P00007515","thread":"main","file":"run-command.c","line":722,"child_id":1,"child_class":"?","use_shell":false,"argv":["git","rev-list","--objects","--stdin","--not","--all","--quiet","--alternate-refs"]}
+{"event":"version","sid":"1234/20230222T072436.291562Z-Ha0f0bee5-P00007515/20230222T072440.148843Z-Ha0f0bee5-P0000751a","thread":"main","time":"2023-02-22T07:24:40.148998Z","file":"common-main.c","line":50,"evt":"3","exe":"2.39.1"}
+{"event":"start","sid":"1234/20230222T072436.291562Z-Ha0f0bee5-P00007515/20230222T072440.148843Z-Ha0f0bee5-P0000751a","thread":"main","file":"common-main.c","line":51,"t_abs":0.003901,"argv":["git","rev-list","--objects","--stdin","--not","--all","--quiet","--alternate-refs"]}
+{"event":"def_repo","sid":"1234/20230222T072436.291562Z-Ha0f0bee5-P00007515/20230222T072440.148843Z-Ha0f0bee5-P0000751a","thread":"main","file":"repository.c","line":136,"repo":1,"worktree":"/gitaly"}
+{"event":"cmd_name","sid":"1234/20230222T072436.291562Z-Ha0f0bee5-P00007515/20230222T072440.148843Z-Ha0f0bee5-P0000751a","thread":"main","file":"git.c","line":461,"name":"rev-list","hierarchy":"fetch/rev-list"}
+{"event":"exit","sid":"1234/20230222T072436.291562Z-Ha0f0bee5-P00007515/20230222T072440.148843Z-Ha0f0bee5-P0000751a","thread":"main","file":"git.c","line":721,"t_abs":0.032566,"code":0}
+{"event":"atexit","sid":"1234/20230222T072436.291562Z-Ha0f0bee5-P00007515/20230222T072440.148843Z-Ha0f0bee5-P0000751a","thread":"main","file":"trace2/tr2_tgt_event.c","line":204,"t_abs":0.032593,"code":0}
+{"event":"child_exit","sid":"1234/20230222T072436.291562Z-Ha0f0bee5-P00007515","thread":"main","file":"run-command.c","line":979,"child_id":1,"pid":29978,"code":0,"t_rel":0.036620}
+{"event":"region_enter","sid":"1234/20230222T072436.291562Z-Ha0f0bee5-P00007515","thread":"main","file":"builtin/fetch.c","line":1353,"repo":1,"nesting":1,"category":"fetch","label":"consume_refs"}
+{"event":"region_leave","sid":"1234/20230222T072436.291562Z-Ha0f0bee5-P00007515","thread":"main","file":"builtin/fetch.c","line":1357,"repo":1,"t_rel":0.000733,"nesting":1,"category":"fetch","label":"consume_refs"}
+{"event":"child_exit","sid":"1234/20230222T072436.291562Z-Ha0f0bee5-P00007515","thread":"main","file":"run-command.c","line":979,"child_id":0,"pid":29974,"code":0,"t_rel":4.245077}
+{"event":"region_enter","sid":"1234/20230222T072436.291562Z-Ha0f0bee5-P00007515","thread":"main","file":"run-command.c","line":1753,"nesting":1,"category":"submodule","label":"parallel/fetch","msg":"max:1"}
+{"event":"region_leave","sid":"1234/20230222T072436.291562Z-Ha0f0bee5-P00007515","thread":"main","file":"run-command.c","line":1790,"t_rel":0.000085,"nesting":1,"category":"submodule","label":"parallel/fetch"}
+{"event":"child_start","sid":"1234/20230222T072436.291562Z-Ha0f0bee5-P00007515","thread":"main","file":"run-command.c","line":722,"child_id":2,"child_class":"?","use_shell":false,"argv":["git","maintenance","run","--auto","--no-quiet"]}
+{"event":"version","sid":"1234/20230222T072436.291562Z-Ha0f0bee5-P00007515/20230222T072440.548471Z-Ha0f0bee5-P0000751b","thread":"main","time":"2023-02-22T07:24:40.548630Z","file":"common-main.c","line":50,"evt":"3","exe":"2.39.1"}
+{"event":"start","sid":"1234/20230222T072436.291562Z-Ha0f0bee5-P00007515/20230222T072440.548471Z-Ha0f0bee5-P0000751b","thread":"main","file":"common-main.c","line":51,"t_abs":0.002892,"argv":["git","maintenance","run","--auto","--no-quiet"]}
+{"event":"def_repo","sid":"1234/20230222T072436.291562Z-Ha0f0bee5-P00007515/20230222T072440.548471Z-Ha0f0bee5-P0000751b","thread":"main","file":"repository.c","line":136,"repo":1,"worktree":"/gitaly"}
+{"event":"cmd_name","sid":"1234/20230222T072436.291562Z-Ha0f0bee5-P00007515/20230222T072440.548471Z-Ha0f0bee5-P0000751b","thread":"main","file":"git.c","line":461,"name":"maintenance","hierarchy":"fetch/maintenance"}
+{"event":"exit","sid":"1234/20230222T072436.291562Z-Ha0f0bee5-P00007515/20230222T072440.548471Z-Ha0f0bee5-P0000751b","thread":"main","file":"git.c","line":721,"t_abs":0.006143,"code":0}
+{"event":"atexit","sid":"1234/20230222T072436.291562Z-Ha0f0bee5-P00007515/20230222T072440.548471Z-Ha0f0bee5-P0000751b","thread":"main","file":"trace2/tr2_tgt_event.c","line":204,"t_abs":0.006160,"code":0}
+{"event":"child_exit","sid":"1234/20230222T072436.291562Z-Ha0f0bee5-P00007515","thread":"main","file":"run-command.c","line":979,"child_id":2,"pid":29979,"code":0,"t_rel":0.011262}
+{"event":"exit","sid":"1234/20230222T072436.291562Z-Ha0f0bee5-P00007515","thread":"main","file":"git.c","line":721,"t_abs":4.262658,"code":0}
+{"event":"atexit","sid":"1234/20230222T072436.291562Z-Ha0f0bee5-P00007515","thread":"main","file":"trace2/tr2_tgt_event.c","line":204,"t_abs":4.262672,"code":0}
diff --git a/internal/trace2/testdata/git-pack-objects.event b/internal/trace2/testdata/git-pack-objects.event
new file mode 100644
index 000000000..710fbd46d
--- /dev/null
+++ b/internal/trace2/testdata/git-pack-objects.event
@@ -0,0 +1,20 @@
+{"event":"version","sid":"1234/20230222T120504.839855Z-Ha0f0bee5-P0000ccc6","thread":"main","time":"2023-02-22T12:05:04.840009Z","file":"common-main.c","line":50,"evt":"3","exe":"2.39.1"}
+{"event":"start","sid":"1234/20230222T120504.839855Z-Ha0f0bee5-P0000ccc6","thread":"main","file":"common-main.c","line":51,"t_abs":0.002338,"argv":["git","pack-objects","toon","--compression=0"]}
+{"event":"def_repo","sid":"1234/20230222T120504.839855Z-Ha0f0bee5-P0000ccc6","thread":"main","file":"repository.c","line":136,"repo":1,"worktree":"/gitaly"}
+{"event":"cmd_name","sid":"1234/20230222T120504.839855Z-Ha0f0bee5-P0000ccc6","thread":"main","file":"git.c","line":461,"name":"pack-objects","hierarchy":"pack-objects"}
+{"event":"region_enter","sid":"1234/20230222T120504.839855Z-Ha0f0bee5-P0000ccc6","thread":"main","file":"builtin/pack-objects.c","line":4460,"repo":1,"nesting":1,"category":"pack-objects","label":"enumerate-objects"}
+{"event":"region_enter","sid":"1234/20230222T120504.839855Z-Ha0f0bee5-P0000ccc6","thread":"main","file":"progress.c","line":268,"repo":1,"nesting":2,"category":"progress","label":"Enumerating objects"}
+{"event":"region_leave","sid":"1234/20230222T120504.839855Z-Ha0f0bee5-P0000ccc6","thread":"main","file":"progress.c","line":346,"repo":1,"t_rel":0.001083,"nesting":2,"category":"progress","label":"Enumerating objects"}
+{"event":"region_leave","sid":"1234/20230222T120504.839855Z-Ha0f0bee5-P0000ccc6","thread":"main","file":"builtin/pack-objects.c","line":4490,"repo":1,"t_rel":0.001435,"nesting":1,"category":"pack-objects","label":"enumerate-objects"}
+{"event":"region_enter","sid":"1234/20230222T120504.839855Z-Ha0f0bee5-P0000ccc6","thread":"main","file":"builtin/pack-objects.c","line":4496,"repo":1,"nesting":1,"category":"pack-objects","label":"prepare-pack"}
+{"event":"region_enter","sid":"1234/20230222T120504.839855Z-Ha0f0bee5-P0000ccc6","thread":"main","file":"progress.c","line":268,"repo":1,"nesting":2,"category":"progress","label":"Counting objects"}
+{"event":"region_leave","sid":"1234/20230222T120504.839855Z-Ha0f0bee5-P0000ccc6","thread":"main","file":"progress.c","line":346,"repo":1,"t_rel":0.000075,"nesting":2,"category":"progress","label":"Counting objects"}
+{"event":"region_leave","sid":"1234/20230222T120504.839855Z-Ha0f0bee5-P0000ccc6","thread":"main","file":"builtin/pack-objects.c","line":4499,"repo":1,"t_rel":0.000090,"nesting":1,"category":"pack-objects","label":"prepare-pack"}
+{"event":"region_enter","sid":"1234/20230222T120504.839855Z-Ha0f0bee5-P0000ccc6","thread":"main","file":"builtin/pack-objects.c","line":4502,"repo":1,"nesting":1,"category":"pack-objects","label":"write-pack-file"}
+{"event":"region_enter","sid":"1234/20230222T120504.839855Z-Ha0f0bee5-P0000ccc6","thread":"main","file":"progress.c","line":268,"repo":1,"nesting":2,"category":"progress","label":"Writing objects"}
+{"event":"region_leave","sid":"1234/20230222T120504.839855Z-Ha0f0bee5-P0000ccc6","thread":"main","file":"progress.c","line":346,"repo":1,"t_rel":0.003972,"nesting":2,"category":"progress","label":"Writing objects"}
+{"event":"data","sid":"1234/20230222T120504.839855Z-Ha0f0bee5-P0000ccc6","thread":"main","file":"builtin/pack-objects.c","line":1312,"repo":1,"t_abs":0.008451,"t_rel":0.003996,"nesting":2,"category":"pack-objects","key":"write_pack_file/wrote","value":"1"}
+{"event":"region_leave","sid":"1234/20230222T120504.839855Z-Ha0f0bee5-P0000ccc6","thread":"main","file":"builtin/pack-objects.c","line":4505,"repo":1,"t_rel":0.004003,"nesting":1,"category":"pack-objects","label":"write-pack-file"}
+{"event":"data","sid":"1234/20230222T120504.839855Z-Ha0f0bee5-P0000ccc6","thread":"main","file":"wrapper.c","line":600,"repo":1,"t_abs":0.008482,"t_rel":0.008482,"nesting":1,"category":"fsync","key":"fsync/writeout-only","value":"2"}
+{"event":"exit","sid":"1234/20230222T120504.839855Z-Ha0f0bee5-P0000ccc6","thread":"main","file":"git.c","line":721,"t_abs":0.008489,"code":0}
+{"event":"atexit","sid":"1234/20230222T120504.839855Z-Ha0f0bee5-P0000ccc6","thread":"main","file":"trace2/tr2_tgt_event.c","line":204,"t_abs":0.008495,"code":0}
diff --git a/internal/trace2/testdata/git-status.event b/internal/trace2/testdata/git-status.event
new file mode 100644
index 000000000..be77a79f7
--- /dev/null
+++ b/internal/trace2/testdata/git-status.event
@@ -0,0 +1,43 @@
+{"event":"version","sid":"1234/20230221T081010.667656Z-Ha0f0bee5-P000006c8","thread":"main","time":"2023-02-21T08:10:10.668546Z","evt":"3","exe":"2.39.1"}
+{"event":"start","sid":"1234/20230221T081010.667656Z-Ha0f0bee5-P000006c8","thread":"main","t_abs":0.086062,"argv":["git","status"]}
+{"event":"def_repo","sid":"1234/20230221T081010.667656Z-Ha0f0bee5-P000006c8","thread":"main","repo":1,"worktree":"/Users/qmnguyen/www/gitlab-development-kit/gitaly"}
+{"event":"cmd_name","sid":"1234/20230221T081010.667656Z-Ha0f0bee5-P000006c8","thread":"main","name":"status","hierarchy":"status"}
+{"event":"region_enter","sid":"1234/20230221T081010.667656Z-Ha0f0bee5-P000006c8","thread":"main","repo":1,"nesting":1,"category":"index","label":"do_read_index","msg":".git/index"}
+{"event":"region_enter","sid":"1234/20230221T081010.667656Z-Ha0f0bee5-P000006c8","thread":"main","repo":1,"nesting":2,"category":"cache_tree","label":"read"}
+{"event":"region_leave","sid":"1234/20230221T081010.667656Z-Ha0f0bee5-P000006c8","thread":"main","repo":1,"t_rel":0.000124,"nesting":2,"category":"cache_tree","label":"read"}
+{"event":"data","sid":"1234/20230221T081010.667656Z-Ha0f0bee5-P000006c8","thread":"main","repo":1,"t_abs":0.089751,"t_rel":0.000452,"nesting":2,"category":"index","key":"read/version","value":"2"}
+{"event":"data","sid":"1234/20230221T081010.667656Z-Ha0f0bee5-P000006c8","thread":"main","repo":1,"t_abs":0.089772,"t_rel":0.000473,"nesting":2,"category":"index","key":"read/cache_nr","value":"1585"}
+{"event":"region_leave","sid":"1234/20230221T081010.667656Z-Ha0f0bee5-P000006c8","thread":"main","repo":1,"t_rel":0.000494,"nesting":1,"category":"index","label":"do_read_index","msg":".git/index"}
+{"event":"region_enter","sid":"1234/20230221T081010.667656Z-Ha0f0bee5-P000006c8","thread":"main","repo":1,"nesting":1,"category":"progress","label":"Refresh index"}
+{"event":"region_enter","sid":"1234/20230221T081010.667656Z-Ha0f0bee5-P000006c8","thread":"main","nesting":2,"category":"index","label":"preload"}
+{"event":"region_leave","sid":"1234/20230221T081010.667656Z-Ha0f0bee5-P000006c8","thread":"main","t_rel":0.004743,"nesting":2,"category":"index","label":"preload"}
+{"event":"region_enter","sid":"1234/20230221T081010.667656Z-Ha0f0bee5-P000006c8","thread":"main","nesting":2,"category":"index","label":"refresh"}
+{"event":"region_leave","sid":"1234/20230221T081010.667656Z-Ha0f0bee5-P000006c8","thread":"main","t_rel":0.000017,"nesting":2,"category":"index","label":"refresh"}
+{"event":"data","sid":"1234/20230221T081010.667656Z-Ha0f0bee5-P000006c8","thread":"main","repo":1,"t_abs":0.094712,"t_rel":0.004818,"nesting":2,"category":"progress","key":"total_objects","value":"1585"}
+{"event":"region_leave","sid":"1234/20230221T081010.667656Z-Ha0f0bee5-P000006c8","thread":"main","repo":1,"t_rel":0.004825,"nesting":1,"category":"progress","label":"Refresh index"}
+{"event":"region_enter","sid":"1234/20230221T081010.667656Z-Ha0f0bee5-P000006c8","thread":"main","repo":1,"nesting":1,"category":"status","label":"worktrees"}
+{"event":"region_enter","sid":"1234/20230221T081010.667656Z-Ha0f0bee5-P000006c8","thread":"main","repo":1,"nesting":2,"category":"diff","label":"setup"}
+{"event":"region_leave","sid":"1234/20230221T081010.667656Z-Ha0f0bee5-P000006c8","thread":"main","repo":1,"t_rel":0.000005,"nesting":2,"category":"diff","label":"setup"}
+{"event":"region_enter","sid":"1234/20230221T081010.667656Z-Ha0f0bee5-P000006c8","thread":"main","repo":1,"nesting":2,"category":"diff","label":"write back to queue"}
+{"event":"region_leave","sid":"1234/20230221T081010.667656Z-Ha0f0bee5-P000006c8","thread":"main","repo":1,"t_rel":0.000004,"nesting":2,"category":"diff","label":"write back to queue"}
+{"event":"region_leave","sid":"1234/20230221T081010.667656Z-Ha0f0bee5-P000006c8","thread":"main","repo":1,"t_rel":0.000072,"nesting":1,"category":"status","label":"worktrees"}
+{"event":"region_enter","sid":"1234/20230221T081010.667656Z-Ha0f0bee5-P000006c8","thread":"main","repo":1,"nesting":1,"category":"status","label":"index"}
+{"event":"region_enter","sid":"1234/20230221T081010.667656Z-Ha0f0bee5-P000006c8","thread":"main","repo":1,"nesting":2,"category":"unpack_trees","label":"unpack_trees"}
+{"event":"region_leave","sid":"1234/20230221T081010.667656Z-Ha0f0bee5-P000006c8","thread":"main","repo":1,"t_rel":0.000079,"nesting":2,"category":"unpack_trees","label":"unpack_trees"}
+{"event":"region_enter","sid":"1234/20230221T081010.667656Z-Ha0f0bee5-P000006c8","thread":"main","repo":1,"nesting":2,"category":"diff","label":"setup"}
+{"event":"region_leave","sid":"1234/20230221T081010.667656Z-Ha0f0bee5-P000006c8","thread":"main","repo":1,"t_rel":0.000003,"nesting":2,"category":"diff","label":"setup"}
+{"event":"region_enter","sid":"1234/20230221T081010.667656Z-Ha0f0bee5-P000006c8","thread":"main","repo":1,"nesting":2,"category":"diff","label":"write back to queue"}
+{"event":"region_leave","sid":"1234/20230221T081010.667656Z-Ha0f0bee5-P000006c8","thread":"main","repo":1,"t_rel":0.000003,"nesting":2,"category":"diff","label":"write back to queue"}
+{"event":"region_leave","sid":"1234/20230221T081010.667656Z-Ha0f0bee5-P000006c8","thread":"main","repo":1,"t_rel":0.001523,"nesting":1,"category":"status","label":"index"}
+{"event":"region_enter","sid":"1234/20230221T081010.667656Z-Ha0f0bee5-P000006c8","thread":"main","repo":1,"nesting":1,"category":"status","label":"untracked"}
+{"event":"region_enter","sid":"1234/20230221T081010.667656Z-Ha0f0bee5-P000006c8","thread":"main","repo":1,"nesting":2,"category":"dir","label":"read_directory"}
+{"event":"region_leave","sid":"1234/20230221T081010.667656Z-Ha0f0bee5-P000006c8","thread":"main","repo":1,"t_rel":0.008231,"nesting":2,"category":"dir","label":"read_directory"}
+{"event":"region_leave","sid":"1234/20230221T081010.667656Z-Ha0f0bee5-P000006c8","thread":"main","repo":1,"t_rel":0.008266,"nesting":1,"category":"status","label":"untracked"}
+{"event":"data","sid":"1234/20230221T081010.667656Z-Ha0f0bee5-P000006c8","thread":"main","repo":1,"t_abs":0.104966,"t_rel":0.104966,"nesting":1,"category":"status","key":"count/changed","value":"0"}
+{"event":"data","sid":"1234/20230221T081010.667656Z-Ha0f0bee5-P000006c8","thread":"main","repo":1,"t_abs":0.104972,"t_rel":0.104972,"nesting":1,"category":"status","key":"count/untracked","value":"1"}
+{"event":"data","sid":"1234/20230221T081010.667656Z-Ha0f0bee5-P000006c8","thread":"main","repo":1,"t_abs":0.104976,"t_rel":0.104976,"nesting":1,"category":"status","key":"count/ignored","value":"0"}
+{"event":"region_enter","sid":"1234/20230221T081010.667656Z-Ha0f0bee5-P000006c8","thread":"main","repo":1,"nesting":1,"category":"status","label":"print"}
+{"event":"region_leave","sid":"1234/20230221T081010.667656Z-Ha0f0bee5-P000006c8","thread":"main","repo":1,"t_rel":0.000205,"nesting":1,"category":"status","label":"print"}
+{"event":"exit","sid":"1234/20230221T081010.667656Z-Ha0f0bee5-P000006c8","thread":"main","t_abs":0.105195,"code":0}
+{"event":"data_json","sid":"1234/20230221T081010.667656Z-Ha0f0bee5-P000006c8","thread":"main","repo":1,"t_abs":0.105204,"t_rel":0.105204,"nesting":1,"category":"traverse_trees","key":"statistics","value":{"traverse_trees_count":1,"traverse_trees_max_depth":1}}
+{"event":"atexit","sid":"1234/20230221T081010.667656Z-Ha0f0bee5-P000006c8","thread":"main","time":"2023-02-21T08:10:10.687877Z","t_abs":0.105208,"code":0}
diff --git a/internal/trace2/trace.go b/internal/trace2/trace.go
new file mode 100644
index 000000000..78ddfe9c7
--- /dev/null
+++ b/internal/trace2/trace.go
@@ -0,0 +1,119 @@
+package trace2
+
+import (
+	"context"
+	"fmt"
+	"sort"
+	"strings"
+	"time"
+)
+
+// Trace denotes a node in the tree representation of Git Trace2 events. A node is not necessary
+// a one-one mapping of an event.
+type Trace struct {
+	// Thread is the name of the thread of the corresponding event. The default thread name is
+	// "main". A new thread is assigned with a new name.
+	Thread string
+	// Name denotes the name of the trace. The node name depends on the event types. Data-type
+	// trace name is the most significant. It can be used to access the accurate data trace node
+	// For example: data:index:refresh/sum_scan
+	Name string
+	// StartTime is the starting time of the trace
+	StartTime time.Time
+	// FinishTime is the starting time of the trace
+	FinishTime time.Time
+	// Metadata is a map of metadata and data extracted from the event. A data-type trace always
+	// stores its data under "data" key of this map
+	Metadata map[string]string
+	// ChildID is the unique ID assigned by the parent process when it spawns a sub-process
+	// The ID of root process is empty.
+	ChildID string
+	// Parent points to the parent node of the current trace. The root node's parent is nil
+	Parent *Trace
+	// Children stores the list of order-significant traces belong to the current trace
+	Children []*Trace
+	// Depth indicates the depth of the trace node
+	Depth int
+}
+
+// IsRoot returns true if the current trace is the root of the tree
+func (trace *Trace) IsRoot() bool {
+	return trace.Parent == nil
+}
+
+// Walk performs in-order tree traversal. It stops at each node and trigger handler function with
+// the current trace.
+func (trace *Trace) Walk(ctx context.Context, handler func(context.Context, *Trace) context.Context) {
+	if trace == nil {
+		return
+	}
+	ctx = handler(ctx, trace)
+	for _, child := range trace.Children {
+		child.Walk(ctx, handler)
+	}
+}
+
+// Inspect returns the formatted string of the tree. It mimics the format for trace2's performance
+// target: https://git-scm.com/docs/api-trace2#_perf_format. It's mostly used for testing and
+// debugging purpose.
+func (trace *Trace) Inspect() string {
+	var output string
+	trace.Walk(context.Background(), func(ctx context.Context, t *Trace) context.Context {
+		line := fmt.Sprintf(
+			"%s | %s | %s | %s%s %s",
+			t.StartTime.Format("03:04:05.000000"),
+			t.FinishTime.Format("03:04:05.000000"),
+			t.fullThread(),
+			strings.Repeat(".", t.Depth),
+			t.Name,
+			t.inspectMetadata(),
+		)
+		if output != "" {
+			output += "\n"
+		}
+		output += strings.TrimSpace(line)
+		return ctx
+	})
+	return output
+}
+
+func (trace *Trace) setName(hints []string) {
+	var parts []string
+	for _, s := range hints {
+		if strings.TrimSpace(s) != "" {
+			parts = append(parts, s)
+		}
+	}
+	trace.Name = strings.Join(parts, ":")
+}
+
+func (trace *Trace) setMetadata(key, value string) {
+	if trace.Metadata == nil {
+		trace.Metadata = map[string]string{}
+	}
+	trace.Metadata[key] = value
+}
+
+func (trace *Trace) fullThread() string {
+	if trace.ChildID != "" {
+		return fmt.Sprintf("%s (child %s)", trace.Thread, trace.ChildID)
+	}
+	return trace.Thread
+}
+
+func (trace *Trace) inspectMetadata() string {
+	var metadata string
+	if len(trace.Metadata) > 0 {
+		var keys []string
+		for key := range trace.Metadata {
+			keys = append(keys, key)
+		}
+		// Sort metadata by key to make output deterministic
+		sort.Strings(keys)
+		for _, key := range keys {
+			metadata += fmt.Sprintf("%s=%q ", key, trace.Metadata[key])
+		}
+		metadata = "(" + strings.TrimSpace(metadata) + ")"
+	}
+	return metadata
+}