Merge branch 'pks-subtransactions-metric' into 'master'

Subtransactions histogram

See merge request gitlab-org/gitaly!2390

5 files changed, 50 insertions, 9 deletions

diff --git a/changelogs/unreleased/pks-subtransactions-metric.yml b/changelogs/unreleased/pks-subtransactions-metric.yml
new file mode 100644
index 000000000..c18939665
--- /dev/null
+++ b/changelogs/unreleased/pks-subtransactions-metric.yml
@@ -0,0 +1,5 @@
+---
+title: Add subtransactions histogram metric
+merge_request: 2390
+author:
+type: added
diff --git a/cmd/praefect/main.go b/cmd/praefect/main.go
index efd65e91d..1553c9c33 100644
--- a/cmd/praefect/main.go
+++ b/cmd/praefect/main.go
@@ -248,9 +248,15 @@ func run(cfgs []starter.Config, conf config.Config) error {
 		return err
 	}
 
+	subtransactionsHistogram, err := metrics.RegisterSubtransactionsHistogram()
+	if err != nil {
+		return err
+	}
+
 	transactionManager := transactions.NewManager(
 		transactions.WithCounterMetric(transactionCounterMetric),
 		transactions.WithDelayMetric(transactionDelayMetric),
+		transactions.WithSubtransactionsMetric(subtransactionsHistogram),
 	)
 
 	var (
diff --git a/internal/praefect/metrics/prometheus.go b/internal/praefect/metrics/prometheus.go
index d552d5f4d..bfff90eef 100644
--- a/internal/praefect/metrics/prometheus.go
+++ b/internal/praefect/metrics/prometheus.go
@@ -82,6 +82,19 @@ func RegisterTransactionDelay(conf promconfig.Config) (metrics.HistogramVec, err
 	return transactionDelay, prometheus.Register(transactionDelay)
 }
 
+// RegisterSubtransactionsHistogram creates and registers a Prometheus counter to
+// gauge the number of subtransactions per transaction.
+func RegisterSubtransactionsHistogram() (metrics.Histogram, error) {
+	subtransactionsHistogram := prometheus.NewHistogram(
+		prometheus.HistogramOpts{
+			Namespace: "gitaly",
+			Subsystem: "praefect",
+			Name:      "subtransactions_per_transaction_total",
+		},
+	)
+	return subtransactionsHistogram, prometheus.Register(subtransactionsHistogram)
+}
+
 var MethodTypeCounter = prometheus.NewCounterVec(
 	prometheus.CounterOpts{
 		Namespace: "gitaly",
diff --git a/internal/praefect/transactions/manager.go b/internal/praefect/transactions/manager.go
index 252146dda..0084eee1f 100644
--- a/internal/praefect/transactions/manager.go
+++ b/internal/praefect/transactions/manager.go
@@ -22,11 +22,12 @@ var ErrNotFound = errors.New("transaction not found")
 // for Praefect to handle transactions directly instead of having to reach out
 // to reference transaction RPCs.
 type Manager struct {
-	txIDGenerator TransactionIDGenerator
-	lock          sync.Mutex
-	transactions  map[uint64]*transaction
-	counterMetric *prometheus.CounterVec
-	delayMetric   metrics.HistogramVec
+	txIDGenerator         TransactionIDGenerator
+	lock                  sync.Mutex
+	transactions          map[uint64]*transaction
+	counterMetric         *prometheus.CounterVec
+	delayMetric           metrics.HistogramVec
+	subtransactionsMetric metrics.Histogram
 }
 
 // TransactionIDGenerator is an interface for types that can generate transaction IDs.
@@ -73,6 +74,13 @@ func WithDelayMetric(delayMetric metrics.HistogramVec) ManagerOpt {
 	}
 }
 
+// WithSubtransactionsMetric is an option to set the subtransactions Prometheus metric
+func WithSubtransactionsMetric(subtransactionsMetric metrics.Histogram) ManagerOpt {
+	return func(mgr *Manager) {
+		mgr.subtransactionsMetric = subtransactionsMetric
+	}
+}
+
 // WithTransactionIDGenerator is an option to set the transaction ID generator
 func WithTransactionIDGenerator(generator TransactionIDGenerator) ManagerOpt {
 	return func(mgr *Manager) {
@@ -83,10 +91,11 @@ func WithTransactionIDGenerator(generator TransactionIDGenerator) ManagerOpt {
 // NewManager creates a new transactions Manager.
 func NewManager(opts ...ManagerOpt) *Manager {
 	mgr := &Manager{
-		txIDGenerator: newTransactionIDGenerator(),
-		transactions:  make(map[uint64]*transaction),
-		counterMetric: prometheus.NewCounterVec(prometheus.CounterOpts{}, []string{"action"}),
-		delayMetric:   prometheus.NewHistogramVec(prometheus.HistogramOpts{}, []string{"action"}),
+		txIDGenerator:         newTransactionIDGenerator(),
+		transactions:          make(map[uint64]*transaction),
+		counterMetric:         prometheus.NewCounterVec(prometheus.CounterOpts{}, []string{"action"}),
+		delayMetric:           prometheus.NewHistogramVec(prometheus.HistogramOpts{}, []string{"action"}),
+		subtransactionsMetric: prometheus.NewHistogram(prometheus.HistogramOpts{}),
 	}
 
 	for _, opt := range opts {
@@ -145,7 +154,11 @@ func (mgr *Manager) RegisterTransaction(ctx context.Context, voters []Voter, thr
 func (mgr *Manager) cancelTransaction(transactionID uint64, transaction *transaction) (map[string]bool, error) {
 	mgr.lock.Lock()
 	defer mgr.lock.Unlock()
+
 	delete(mgr.transactions, transactionID)
+
+	mgr.subtransactionsMetric.Observe(float64(transaction.countSubtransactions()))
+
 	return transaction.cancel(), nil
 }
 
diff --git a/internal/praefect/transactions/transaction.go b/internal/praefect/transactions/transaction.go
index cfcca1658..feb215c2c 100644
--- a/internal/praefect/transactions/transaction.go
+++ b/internal/praefect/transactions/transaction.go
@@ -103,6 +103,10 @@ func (t *transaction) cancel() map[string]bool {
 	return results
 }
 
+func (t *transaction) countSubtransactions() int {
+	return len(t.subtransactions)
+}
+
 // getOrCreateSubtransaction gets an ongoing subtransaction on which the given
 // node hasn't yet voted on or creates a new one if the node has succeeded on
 // all subtransactions. In case the node has failed on any of the