Add gitaly-debug production debugging tool

4 files changed, 175 insertions, 0 deletions

diff --git a/.gitignore b/.gitignore
index 5799f37c0..0c41b8995 100644
--- a/.gitignore
+++ b/.gitignore
@@ -13,3 +13,4 @@
 /ruby/tmp
 *.socket
 git-env
+/gitaly-debug
diff --git a/changelogs/unreleased/gitaly-debug.yml b/changelogs/unreleased/gitaly-debug.yml
new file mode 100644
index 000000000..c0b940641
--- /dev/null
+++ b/changelogs/unreleased/gitaly-debug.yml
@@ -0,0 +1,5 @@
+---
+title: Add gitaly-debug production debugging tool
+merge_request: 967
+author:
+type: added
diff --git a/cmd/gitaly-debug/README.md b/cmd/gitaly-debug/README.md
new file mode 100644
index 000000000..abbddfb8c
--- /dev/null
+++ b/cmd/gitaly-debug/README.md
@@ -0,0 +1,34 @@
+# gitaly-debug
+
+Gitaly-debug provides "production debugging" tools for Gitaly and Git
+performance. It is intended to help production engineers and support
+engineers investigate Gitaly performance problems.
+
+## Installation
+
+If you're using GitLab 11.6 or newer this tool should be installed on
+your GitLab / Gitaly server already at
+`/opt/gitlab/embedded/bin/gitaly-debug`.
+
+If you're investigating an older GitLab version you can compile this
+tool offline and copy the executable to your server.
+
+    GOOS=linux GOARCH=amd64 go build -o gitaly-debug
+
+## Subcommands
+
+### `simulate-http-clone`
+
+    gitaly-debug simulate-http-clone /path/to/repo.git
+
+This simulates the server workload for a full HTTP Git clone on a
+repository on your Gitaly server. You can use this to determine the
+best-case performance of `git clone`. An example application is to
+determine if a slow `git clone` is bottle-necked by Gitaly server
+performance, or by something downstream.
+
+The results returned by this command give an indication of the ideal
+case performance of a `git clone` on the repository, as if there is
+unlimited network bandwidth and no latency. This speed will not be
+reached in real life but it shows the best you can hope for from a given
+repository on a given Gitaly server.
diff --git a/cmd/gitaly-debug/main.go b/cmd/gitaly-debug/main.go
new file mode 100644
index 000000000..57d16e45d
--- /dev/null
+++ b/cmd/gitaly-debug/main.go
@@ -0,0 +1,135 @@
+package main
+
+import (
+	"bufio"
+	"bytes"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"regexp"
+	"time"
+)
+
+const (
+	usage = `Usage: gitaly-debug SUBCOMMAND ARGS
+
+Subcommands:
+
+simulate-http-clone GIT_DIR
+	Simulates the server side workload of serving a full git clone over
+	HTTP. The clone data is written to /dev/null. Note that in real life
+	the workload also depends on the transport capabilities requested by
+	the client; this tool uses a fixed set of capabilities.
+`
+)
+
+func main() {
+	if len(os.Args) != 3 {
+		fatal(usage)
+	}
+	gitDir := os.Args[2]
+
+	switch os.Args[1] {
+	case "simulate-http-clone":
+		testHTTPCloneSpeed(gitDir)
+	default:
+		fatal(usage)
+	}
+}
+
+func testHTTPCloneSpeed(gitDir string) {
+	msg("Generating server response for HTTP clone. Data goes to /dev/null.")
+	infoRefs := exec.Command("git", "upload-pack", "--stateless-rpc", "--advertise-refs", gitDir)
+	infoRefs.Stderr = os.Stderr
+	out, err := infoRefs.StdoutPipe()
+	noError(err)
+
+	start := time.Now()
+	noError(infoRefs.Start())
+
+	infoScanner := bufio.NewScanner(out)
+	var infoLines []string
+	for infoScanner.Scan() {
+		infoLines = append(infoLines, infoScanner.Text())
+	}
+	noError(infoScanner.Err())
+
+	noError(infoRefs.Wait())
+
+	msg("simulated GET \"/info/refs?service=git-upload-pack\" returned %d lines, took %v", len(infoLines), time.Since(start))
+
+	if len(infoLines) == 0 {
+		fatal("no refs were advertised")
+	}
+
+	request := &bytes.Buffer{}
+	refsHeads := regexp.MustCompile(`^[a-f0-9]{44} refs/heads/`)
+	firstLine := true
+	for _, line := range infoLines {
+		if !refsHeads.MatchString(line) {
+			continue
+		}
+
+		commitID := line[4:44]
+
+		if firstLine {
+			firstLine = false
+			fmt.Fprintf(request, "0098want %s multi_ack_detailed no-done side-band-64k thin-pack ofs-delta deepen-since deepen-not agent=git/2.19.1\n", commitID)
+			continue
+		}
+
+		fmt.Fprintf(request, "0032want %s\n", commitID)
+	}
+	fmt.Fprint(request, "00000009done\n")
+
+	uploadPack := exec.Command("git", "upload-pack", "--stateless-rpc", gitDir)
+	uploadPack.Stdin = request
+	uploadPack.Stderr = os.Stderr
+	out, err = uploadPack.StdoutPipe()
+	noError(err)
+
+	start = time.Now()
+	noError(uploadPack.Start())
+
+	n, err := io.Copy(ioutil.Discard, out)
+	noError(err)
+
+	msg("simulated POST \"/git-upload-pack\" returned %s, took %v", humanBytes(n), time.Since(start))
+}
+
+func noError(err error) {
+	if err != nil {
+		fatal(err)
+	}
+}
+
+func fatal(a interface{}) {
+	msg("%v", a)
+	os.Exit(1)
+}
+
+func msg(format string, a ...interface{}) {
+	fmt.Fprintln(os.Stderr, fmt.Sprintf(format, a...))
+}
+
+func humanBytes(n int64) string {
+	units := []struct {
+		size  int64
+		label string
+	}{
+		{size: 1000000000000, label: "TB"},
+		{size: 1000000000, label: "GB"},
+		{size: 1000000, label: "MB"},
+		{size: 1000, label: "KB"},
+	}
+
+	for _, u := range units {
+		if n > u.size {
+			return fmt.Sprintf("%.2f %s", float32(n)/float32(u.size), u.label)
+		}
+	}
+
+	return fmt.Sprintf("%d bytes", n)
+}