transactions: Track the number of subtransactions created

With our new support for subtransactions, we cannot quite tell how many
votes there are per created transaction. While we do know how many
subtransactions there are, we cannot correlate a single transaction with
its subtransactions. This is quite an interesting metric though to see
how often the reference-transaction hook gets executed per transaction
and to better estimate the overhead of reaching out to the transaction
manager repeatedly.

Let's add a new histgoram that allows us to record the number of
subtransactions per transaction.

2 files changed, 26 insertions, 9 deletions

diff --git a/internal/praefect/transactions/manager.go b/internal/praefect/transactions/manager.go
index 252146dda..0084eee1f 100644
--- a/internal/praefect/transactions/manager.go
+++ b/internal/praefect/transactions/manager.go
@@ -22,11 +22,12 @@ var ErrNotFound = errors.New("transaction not found")
 // for Praefect to handle transactions directly instead of having to reach out
 // to reference transaction RPCs.
 type Manager struct {
-	txIDGenerator TransactionIDGenerator
-	lock          sync.Mutex
-	transactions  map[uint64]*transaction
-	counterMetric *prometheus.CounterVec
-	delayMetric   metrics.HistogramVec
+	txIDGenerator         TransactionIDGenerator
+	lock                  sync.Mutex
+	transactions          map[uint64]*transaction
+	counterMetric         *prometheus.CounterVec
+	delayMetric           metrics.HistogramVec
+	subtransactionsMetric metrics.Histogram
 }
 
 // TransactionIDGenerator is an interface for types that can generate transaction IDs.
@@ -73,6 +74,13 @@ func WithDelayMetric(delayMetric metrics.HistogramVec) ManagerOpt {
 	}
 }
 
+// WithSubtransactionsMetric is an option to set the subtransactions Prometheus metric
+func WithSubtransactionsMetric(subtransactionsMetric metrics.Histogram) ManagerOpt {
+	return func(mgr *Manager) {
+		mgr.subtransactionsMetric = subtransactionsMetric
+	}
+}
+
 // WithTransactionIDGenerator is an option to set the transaction ID generator
 func WithTransactionIDGenerator(generator TransactionIDGenerator) ManagerOpt {
 	return func(mgr *Manager) {
@@ -83,10 +91,11 @@ func WithTransactionIDGenerator(generator TransactionIDGenerator) ManagerOpt {
 // NewManager creates a new transactions Manager.
 func NewManager(opts ...ManagerOpt) *Manager {
 	mgr := &Manager{
-		txIDGenerator: newTransactionIDGenerator(),
-		transactions:  make(map[uint64]*transaction),
-		counterMetric: prometheus.NewCounterVec(prometheus.CounterOpts{}, []string{"action"}),
-		delayMetric:   prometheus.NewHistogramVec(prometheus.HistogramOpts{}, []string{"action"}),
+		txIDGenerator:         newTransactionIDGenerator(),
+		transactions:          make(map[uint64]*transaction),
+		counterMetric:         prometheus.NewCounterVec(prometheus.CounterOpts{}, []string{"action"}),
+		delayMetric:           prometheus.NewHistogramVec(prometheus.HistogramOpts{}, []string{"action"}),
+		subtransactionsMetric: prometheus.NewHistogram(prometheus.HistogramOpts{}),
 	}
 
 	for _, opt := range opts {
@@ -145,7 +154,11 @@ func (mgr *Manager) RegisterTransaction(ctx context.Context, voters []Voter, thr
 func (mgr *Manager) cancelTransaction(transactionID uint64, transaction *transaction) (map[string]bool, error) {
 	mgr.lock.Lock()
 	defer mgr.lock.Unlock()
+
 	delete(mgr.transactions, transactionID)
+
+	mgr.subtransactionsMetric.Observe(float64(transaction.countSubtransactions()))
+
 	return transaction.cancel(), nil
 }
 
diff --git a/internal/praefect/transactions/transaction.go b/internal/praefect/transactions/transaction.go
index cfcca1658..feb215c2c 100644
--- a/internal/praefect/transactions/transaction.go
+++ b/internal/praefect/transactions/transaction.go
@@ -103,6 +103,10 @@ func (t *transaction) cancel() map[string]bool {
 	return results
 }
 
+func (t *transaction) countSubtransactions() int {
+	return len(t.subtransactions)
+}
+
 // getOrCreateSubtransaction gets an ongoing subtransaction on which the given
 // node hasn't yet voted on or creates a new one if the node has succeeded on
 // all subtransactions. In case the node has failed on any of the