Merge branch 'jliu-server-side-backup-metrics' into 'master'

backup: Add server-side backup metrics

See merge request https://gitlab.com/gitlab-org/gitaly/-/merge_requests/6540

Merged-by: James Fargher <jfargher@gitlab.com>
Approved-by: Ahmad Sherif <ahmad@gitlab.com>
Approved-by: James Fargher <jfargher@gitlab.com>
Reviewed-by: James Fargher <jfargher@gitlab.com>
Reviewed-by: Ahmad Sherif <ahmad@gitlab.com>
Co-authored-by: James Liu <jliu@gitlab.com>

3 files changed, 40 insertions, 3 deletions

diff --git a/internal/backup/backup.go b/internal/backup/backup.go
index e58fc8718..4b05c9aba 100644
--- a/internal/backup/backup.go
+++ b/internal/backup/backup.go
@@ -7,9 +7,12 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"math"
 	"strings"
 	"time"
 
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
 	"gitlab.com/gitlab-org/gitaly/v16/internal/git"
 	"gitlab.com/gitlab-org/gitaly/v16/internal/git/catfile"
 	"gitlab.com/gitlab-org/gitaly/v16/internal/git/localrepo"
@@ -28,6 +31,19 @@ var (
 	ErrSkipped = errors.New("repository skipped")
 	// ErrDoesntExist means that the data was not found.
 	ErrDoesntExist = errors.New("doesn't exist")
+
+	backupLatency = promauto.NewHistogramVec(
+		prometheus.HistogramOpts{
+			Name: "gitaly_backup_latency_seconds",
+			Help: "Latency of a repository backup by phase",
+		},
+		[]string{"phase"})
+	backupBundleSize = promauto.NewHistogram(
+		prometheus.HistogramOpts{
+			Name:    "gitaly_backup_bundle_bytes",
+			Help:    "Size of a Git bundle uploaded in a backup",
+			Buckets: prometheus.ExponentialBucketsRange(1, 10*math.Pow(1024, 3), 20), // up to 10GB
+		})
 )
 
 // Sink is an abstraction over the real storage used for storing/restoring backups.
@@ -369,6 +385,9 @@ func setContextServerInfo(ctx context.Context, server *storage.ServerInfo, stora
 }
 
 func (mgr *Manager) writeBundle(ctx context.Context, repo Repository, step *Step) (returnErr error) {
+	timer := prometheus.NewTimer(backupLatency.WithLabelValues("bundle"))
+	defer timer.ObserveDuration()
+
 	var patterns io.Reader
 	if len(step.PreviousRefPath) > 0 {
 		// If there is a previous ref path, then we are creating an increment
@@ -389,6 +408,7 @@ func (mgr *Manager) writeBundle(ctx context.Context, repo Repository, step *Step
 		return mgr.sink.GetWriter(ctx, step.BundlePath)
 	})
 	defer func() {
+		backupBundleSize.Observe(float64(w.BytesWritten()))
 		if err := w.Close(); err != nil && returnErr == nil {
 			returnErr = fmt.Errorf("write bundle: %w", err)
 		}
@@ -481,6 +501,9 @@ func (mgr *Manager) restoreBundle(ctx context.Context, repo Repository, path str
 }
 
 func (mgr *Manager) writeCustomHooks(ctx context.Context, repo Repository, path string) (returnErr error) {
+	timer := prometheus.NewTimer(backupLatency.WithLabelValues("custom_hooks"))
+	defer timer.ObserveDuration()
+
 	w := NewLazyWriter(func() (io.WriteCloser, error) {
 		return mgr.sink.GetWriter(ctx, path)
 	})
@@ -514,6 +537,9 @@ func (mgr *Manager) restoreCustomHooks(ctx context.Context, repo Repository, pat
 // writeRefs writes the previously fetched list of refs in the same output
 // format as `git-show-ref(1)`
 func (mgr *Manager) writeRefs(ctx context.Context, path string, refs []git.Reference) (returnErr error) {
+	timer := prometheus.NewTimer(backupLatency.WithLabelValues("refs"))
+	defer timer.ObserveDuration()
+
 	w, err := mgr.sink.GetWriter(ctx, path)
 	if err != nil {
 		return fmt.Errorf("write refs: %w", err)
diff --git a/internal/backup/lazy.go b/internal/backup/lazy.go
index 7c38d5c6b..a66b90621 100644
--- a/internal/backup/lazy.go
+++ b/internal/backup/lazy.go
@@ -8,8 +8,9 @@ import (
 // Write. This means it will only create a file if there will be data written
 // to it.
 type LazyWriter struct {
-	create func() (io.WriteCloser, error)
-	w      io.WriteCloser
+	create       func() (io.WriteCloser, error)
+	w            io.WriteCloser
+	bytesWritten int
 }
 
 // NewLazyWriter initializes a new LazyWriter. create is called on the first
@@ -20,6 +21,12 @@ func NewLazyWriter(create func() (io.WriteCloser, error)) *LazyWriter {
 	}
 }
 
+// BytesWritten returns the total number of bytes written to the underlying
+// WriteCloser. The count is never explicitly reset to 0.
+func (w *LazyWriter) BytesWritten() int {
+	return w.bytesWritten
+}
+
 func (w *LazyWriter) Write(p []byte) (int, error) {
 	if w.w == nil {
 		var err error
@@ -29,7 +36,9 @@ func (w *LazyWriter) Write(p []byte) (int, error) {
 		}
 	}
 
-	return w.w.Write(p)
+	n, err := w.w.Write(p)
+	w.bytesWritten += n
+	return n, err
 }
 
 // Close calls Close on the WriteCloser returned by Create, passing on any
diff --git a/internal/backup/lazy_test.go b/internal/backup/lazy_test.go
index 5f4ff514f..15cce1075 100644
--- a/internal/backup/lazy_test.go
+++ b/internal/backup/lazy_test.go
@@ -36,6 +36,7 @@ func TestLazyWriter(t *testing.T) {
 		_, err = io.Copy(w, iotest.OneByteReader(bytes.NewReader(expectedData)))
 		require.NoError(t, err)
 		require.FileExists(t, tempFilePath)
+		require.Equal(t, len(expectedData), w.BytesWritten())
 
 		data, err := os.ReadFile(tempFilePath)
 		require.NoError(t, err)
@@ -50,5 +51,6 @@ func TestLazyWriter(t *testing.T) {
 		n, err := w.Write(make([]byte, 100))
 		require.Equal(t, 0, n)
 		require.Equal(t, assert.AnError, err)
+		require.Equal(t, 0, w.BytesWritten())
 	})
 }