transactions: Add histogram to count voters

While we already track how many actions on a transaction are performed,
it's currently impossible to say how many voters are part of a single
transaction. This information may be quite important to asses current
state of a cluster.

Add a new histogram "gitaly_praefect_voters_per_transaction_total" to
keep track of this.

3 files changed, 61 insertions, 7 deletions

diff --git a/cmd/praefect/main.go b/cmd/praefect/main.go
index 2cba3aac7..efac1d21b 100644
--- a/cmd/praefect/main.go
+++ b/cmd/praefect/main.go
@@ -263,6 +263,11 @@ func run(cfgs []starter.Config, conf config.Config) error {
 		transactions.WithSubtransactionsMetric(subtransactionsHistogram),
 	)
 
+	transactionVoters, err := metrics.RegisterTransactionVoters(conf)
+	if err != nil {
+		return err
+	}
+
 	var (
 		// top level server dependencies
 		coordinator = praefect.NewCoordinator(
@@ -272,6 +277,7 @@ func run(cfgs []starter.Config, conf config.Config) error {
 			transactionManager,
 			conf,
 			protoregistry.GitalyProtoPreregistered,
+			praefect.WithVotersMetric(transactionVoters),
 		)
 		repl = praefect.NewReplMgr(
 			logger,
diff --git a/internal/praefect/coordinator.go b/internal/praefect/coordinator.go
index b2c8a574b..48c1dec25 100644
--- a/internal/praefect/coordinator.go
+++ b/internal/praefect/coordinator.go
@@ -7,6 +7,7 @@ import (
 
 	"github.com/golang/protobuf/proto"
 	"github.com/grpc-ecosystem/go-grpc-middleware/logging/logrus/ctxlogrus"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/sirupsen/logrus"
 	"gitlab.com/gitlab-org/gitaly/internal/helper"
 	"gitlab.com/gitlab-org/gitaly/internal/metadata/featureflag"
@@ -19,6 +20,7 @@ import (
 	"gitlab.com/gitlab-org/gitaly/internal/praefect/nodes"
 	"gitlab.com/gitlab-org/gitaly/internal/praefect/protoregistry"
 	"gitlab.com/gitlab-org/gitaly/internal/praefect/transactions"
+	prommetrics "gitlab.com/gitlab-org/gitaly/internal/prometheus/metrics"
 	"gitlab.com/gitlab-org/gitaly/proto/go/gitalypb"
 	"gitlab.com/gitlab-org/labkit/correlation"
 	"golang.org/x/sync/errgroup"
@@ -74,12 +76,13 @@ type grpcCall struct {
 // downstream server. The coordinator is thread safe; concurrent calls to
 // register nodes are safe.
 type Coordinator struct {
-	nodeMgr  nodes.Manager
-	txMgr    *transactions.Manager
-	queue    datastore.ReplicationEventQueue
-	rs       datastore.RepositoryStore
-	registry *protoregistry.Registry
-	conf     config.Config
+	nodeMgr      nodes.Manager
+	txMgr        *transactions.Manager
+	queue        datastore.ReplicationEventQueue
+	rs           datastore.RepositoryStore
+	registry     *protoregistry.Registry
+	conf         config.Config
+	votersMetric prommetrics.HistogramVec
 }
 
 // NewCoordinator returns a new Coordinator that utilizes the provided logger
@@ -90,14 +93,35 @@ func NewCoordinator(
 	txMgr *transactions.Manager,
 	conf config.Config,
 	r *protoregistry.Registry,
+	opts ...CoordinatorOpt,
 ) *Coordinator {
-	return &Coordinator{
+	coordinator := &Coordinator{
 		queue:    queue,
 		rs:       rs,
 		registry: r,
 		nodeMgr:  nodeMgr,
 		txMgr:    txMgr,
 		conf:     conf,
+		votersMetric: prometheus.NewHistogramVec(
+			prometheus.HistogramOpts{},
+			[]string{"virtual_storage"},
+		),
+	}
+
+	for _, opt := range opts {
+		opt(coordinator)
+	}
+
+	return coordinator
+}
+
+// CoordinatorOpt is a self referential option for Coordinator
+type CoordinatorOpt func(*Coordinator)
+
+// WithVotersMetric is an option to set the voters Prometheus metric
+func WithVotersMetric(votersMetric prommetrics.HistogramVec) CoordinatorOpt {
+	return func(coordinator *Coordinator) {
+		coordinator.votersMetric = votersMetric
 	}
 }
 
@@ -275,6 +299,8 @@ func (c *Coordinator) mutatorStreamParameters(ctx context.Context, call grpcCall
 			}
 		}
 
+		c.votersMetric.WithLabelValues(virtualStorage).Observe(float64(1 + len(participatingSecondaries)))
+
 		transaction, transactionCleanup, err := c.registerTransaction(ctx, shard.Primary, participatingSecondaries)
 		if err != nil {
 			return nil, err
diff --git a/internal/praefect/metrics/prometheus.go b/internal/praefect/metrics/prometheus.go
index 4a1f4d6b2..f06d95614 100644
--- a/internal/praefect/metrics/prometheus.go
+++ b/internal/praefect/metrics/prometheus.go
@@ -3,6 +3,7 @@ package metrics
 import (
 	"github.com/prometheus/client_golang/prometheus"
 	promconfig "gitlab.com/gitlab-org/gitaly/internal/config/prometheus"
+	"gitlab.com/gitlab-org/gitaly/internal/praefect/config"
 	"gitlab.com/gitlab-org/gitaly/internal/prometheus/metrics"
 )
 
@@ -95,6 +96,27 @@ func RegisterSubtransactionsHistogram() (metrics.Histogram, error) {
 	return subtransactionsHistogram, prometheus.Register(subtransactionsHistogram)
 }
 
+// RegisterTransactionVoters creates and registers a Prometheus counter to gauge
+// the number of voters per transaction.
+func RegisterTransactionVoters(conf config.Config) (metrics.HistogramVec, error) {
+	maxVoters := 0
+	for _, storage := range conf.VirtualStorages {
+		if len(storage.Nodes) > maxVoters {
+			maxVoters = len(storage.Nodes)
+		}
+	}
+
+	transactionVoters := prometheus.NewHistogramVec(
+		prometheus.HistogramOpts{
+			Name:    "gitaly_praefect_voters_per_transaction_total",
+			Help:    "The number of voters a given transaction was created with",
+			Buckets: prometheus.LinearBuckets(0, 1, maxVoters),
+		},
+		[]string{"virtual_storage"},
+	)
+	return transactionVoters, prometheus.Register(transactionVoters)
+}
+
 var MethodTypeCounter = prometheus.NewCounterVec(
 	prometheus.CounterOpts{
 		Namespace: "gitaly",