hook: Measure delay introduced by voting on transactions

While we already measure how long it takes until a transaction reaches
consensus on the server-side, we don't do so for the client-side. As
there may be additional latency involved due to having to connect to
Praefect first and doing an RPC, we should have both metrics.

Create a new metric "gitaly_reference_transaction_voting_delay_seconds"
to measure this on client-side.

4 files changed, 47 insertions, 9 deletions

diff --git a/changelogs/unreleased/pks-transaction-metrics.yml b/changelogs/unreleased/pks-transaction-metrics.yml
new file mode 100644
index 000000000..82951f1a2
--- /dev/null
+++ b/changelogs/unreleased/pks-transaction-metrics.yml
@@ -0,0 +1,5 @@
+---
+title: Improve transaction metrics
+merge_request: 2441
+author:
+type: added
diff --git a/internal/service/hook/pre_receive.go b/internal/service/hook/pre_receive.go
index c6f500edd..06d7ecd98 100644
--- a/internal/service/hook/pre_receive.go
+++ b/internal/service/hook/pre_receive.go
@@ -13,6 +13,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/prometheus/client_golang/prometheus"
 	"gitlab.com/gitlab-org/gitaly/internal/config"
 	"gitlab.com/gitlab-org/gitaly/internal/git/hooks"
 	"gitlab.com/gitlab-org/gitaly/internal/gitlabshell"
@@ -66,6 +67,8 @@ func (s *server) getPraefectConn(ctx context.Context, server *metadata.PraefectS
 }
 
 func (s *server) voteOnTransaction(ctx context.Context, hash []byte, env []string) error {
+	defer prometheus.NewTimer(s.votingDelayMetric).ObserveDuration()
+
 	tx, err := metadata.TransactionFromEnv(env)
 	if err != nil {
 		if errors.Is(err, metadata.ErrTransactionNotFound) {
diff --git a/internal/service/hook/server.go b/internal/service/hook/server.go
index 2685417d0..ce3dc6f2e 100644
--- a/internal/service/hook/server.go
+++ b/internal/service/hook/server.go
@@ -1,22 +1,40 @@
 package hook
 
 import (
+	"github.com/prometheus/client_golang/prometheus"
 	"gitlab.com/gitlab-org/gitaly/client"
 	"gitlab.com/gitlab-org/gitaly/internal/config"
 	"gitlab.com/gitlab-org/gitaly/proto/go/gitalypb"
 )
 
 type server struct {
-	conns       *client.Pool
-	hooksConfig config.Hooks
-	gitlabAPI   GitlabAPI
+	conns             *client.Pool
+	hooksConfig       config.Hooks
+	gitlabAPI         GitlabAPI
+	votingDelayMetric prometheus.Histogram
 }
 
 // NewServer creates a new instance of a gRPC namespace server
-func NewServer(gitlab GitlabAPI, hooksConfig config.Hooks) gitalypb.HookServiceServer {
-	return &server{
-		gitlabAPI:   gitlab,
-		hooksConfig: hooksConfig,
-		conns:       client.NewPool(),
+func NewServer(gitlab GitlabAPI, hooksConfig config.Hooks, serverOpts ...ServerOpt) gitalypb.HookServiceServer {
+	s := &server{
+		gitlabAPI:         gitlab,
+		hooksConfig:       hooksConfig,
+		conns:             client.NewPool(),
+		votingDelayMetric: prometheus.NewHistogram(prometheus.HistogramOpts{}),
+	}
+
+	for _, serverOpt := range serverOpts {
+		serverOpt(s)
+	}
+
+	return s
+}
+
+// ServerOpt is a self referential option for server
+type ServerOpt func(s *server)
+
+func WithVotingDelayMetric(metric prometheus.Histogram) ServerOpt {
+	return func(s *server) {
+		s.votingDelayMetric = metric
 	}
 }
diff --git a/internal/service/register.go b/internal/service/register.go
index 5e20aa7cf..e0bafdff7 100644
--- a/internal/service/register.go
+++ b/internal/service/register.go
@@ -49,6 +49,14 @@ var (
 		},
 		[]string{"git_negotiation_feature"},
 	)
+
+	votingDelayMetric = promauto.NewHistogram(
+		prometheus.HistogramOpts{
+			Name:    "gitaly_hook_transaction_voting_delay_seconds",
+			Help:    "Delay between calling out to transaction service and receiving a response",
+			Buckets: config.Config.Prometheus.GRPCLatencyBuckets,
+		},
+	)
 )
 
 // RegisterAll will register all the known grpc services with
@@ -73,7 +81,11 @@ func RegisterAll(grpcServer *grpc.Server, cfg config.Cfg, rubyServer *rubyserver
 	gitalypb.RegisterRemoteServiceServer(grpcServer, remote.NewServer(rubyServer))
 	gitalypb.RegisterServerServiceServer(grpcServer, server.NewServer(cfg.Storages))
 	gitalypb.RegisterObjectPoolServiceServer(grpcServer, objectpool.NewServer(locator))
-	gitalypb.RegisterHookServiceServer(grpcServer, hook.NewServer(gitlabAPI, cfg.Hooks))
+	gitalypb.RegisterHookServiceServer(grpcServer, hook.NewServer(
+		gitlabAPI,
+		cfg.Hooks,
+		hook.WithVotingDelayMetric(votingDelayMetric),
+	))
 	gitalypb.RegisterInternalGitalyServer(grpcServer, internalgitaly.NewServer(cfg.Storages))
 
 	healthpb.RegisterHealthServer(grpcServer, health.NewServer())