Merge branch 'kn-4545-replication-metrics' into 'master'

metrics: instantiate replication metrics with 0 values

Closes #4545

See merge request https://gitlab.com/gitlab-org/gitaly/-/merge_requests/5157

Merged-by: Toon Claes <toon@gitlab.com>
Approved-by: Quang-Minh Nguyen <qmnguyen@gitlab.com>
Approved-by: Toon Claes <toon@gitlab.com>
Reviewed-by: Quang-Minh Nguyen <qmnguyen@gitlab.com>
Co-authored-by: Karthik Nayak <knayak@gitlab.com>

3 files changed, 82 insertions, 0 deletions

diff --git a/internal/praefect/datastore/datastore.go b/internal/praefect/datastore/datastore.go
index 48e9df648..dfa9d586e 100644
--- a/internal/praefect/datastore/datastore.go
+++ b/internal/praefect/datastore/datastore.go
@@ -36,6 +36,7 @@ const (
 // ChangeType indicates what kind of change the replication is propagating
 type ChangeType string
 
+// Any fields added here should also be added below to GetAllChangeTypes
 const (
 	// UpdateRepo is when a replication updates a repository in place
 	UpdateRepo = ChangeType("update")
@@ -49,6 +50,12 @@ const (
 	RenameRepo = ChangeType("rename")
 )
 
+// GetAllChangeTypes is used to define and provide all the various ChangeType
+// constants. This is useful to iterate over and set labels in metrics.
+func GetAllChangeTypes() []ChangeType {
+	return []ChangeType{UpdateRepo, CreateRepo, DeleteRepo, DeleteReplica, RenameRepo}
+}
+
 func (ct ChangeType) String() string {
 	return string(ct)
 }
diff --git a/internal/praefect/metrics/prometheus.go b/internal/praefect/metrics/prometheus.go
index 6449a883f..32ac0bcf1 100644
--- a/internal/praefect/metrics/prometheus.go
+++ b/internal/praefect/metrics/prometheus.go
@@ -1,9 +1,12 @@
 package metrics
 
 import (
+	"fmt"
+
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promauto"
 	gitalycfgprom "gitlab.com/gitlab-org/gitaly/v15/internal/gitaly/config/prometheus"
+	"gitlab.com/gitlab-org/gitaly/v15/internal/praefect/datastore"
 	"gitlab.com/gitlab-org/gitaly/v15/internal/prometheus/metrics"
 )
 
@@ -20,6 +23,15 @@ func RegisterReplicationDelay(conf gitalycfgprom.Config, registerer prometheus.R
 		[]string{"type"},
 	)
 
+	for _, changeType := range datastore.GetAllChangeTypes() {
+		// We don't care about the observer here, we only want to initialize the
+		// metric with the various label values
+		_, err := replicationDelay.GetMetricWithLabelValues(changeType.String())
+		if err != nil {
+			return replicationDelay, fmt.Errorf("adding default label value %s failed: %w", changeType, err)
+		}
+	}
+
 	return replicationDelay, registerer.Register(replicationDelay)
 }
 
@@ -36,6 +48,15 @@ func RegisterReplicationLatency(conf gitalycfgprom.Config, registerer prometheus
 		[]string{"type"},
 	)
 
+	for _, changeType := range datastore.GetAllChangeTypes() {
+		// We don't care about the observer here, we only want to initialize the
+		// metric with the various label values
+		_, err := replicationLatency.GetMetricWithLabelValues(changeType.String())
+		if err != nil {
+			return replicationLatency, fmt.Errorf("adding default label value %s failed: %w", changeType, err)
+		}
+	}
+
 	return replicationLatency, registerer.Register(replicationLatency)
 }
 
diff --git a/internal/praefect/metrics/prometheus_test.go b/internal/praefect/metrics/prometheus_test.go
new file mode 100644
index 000000000..ec45bf720
--- /dev/null
+++ b/internal/praefect/metrics/prometheus_test.go
@@ -0,0 +1,54 @@
+package metrics
+
+import (
+	"testing"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/stretchr/testify/require"
+	gitalycfgprom "gitlab.com/gitlab-org/gitaly/v15/internal/gitaly/config/prometheus"
+	"gitlab.com/gitlab-org/gitaly/v15/internal/praefect/datastore"
+)
+
+func TestRegisterReplicationDelay(t *testing.T) {
+	t.Parallel()
+
+	registry := prometheus.NewRegistry()
+	_, err := RegisterReplicationDelay(gitalycfgprom.Config{
+		GRPCLatencyBuckets: prometheus.LinearBuckets(0, 2, 10),
+	}, registry)
+	require.NoError(t, err)
+
+	// We get the metric family, it should be instantiated with the 0 values
+	mfs, err := registry.Gather()
+	require.NoError(t, err)
+
+	// We only expect one metric family
+	require.Len(t, mfs, 1)
+
+	metricFamily := mfs[0]
+	require.Equal(t, "gitaly_praefect_replication_delay", metricFamily.GetName())
+	// The number of metrics should be equal to number of label values
+	require.Len(t, metricFamily.Metric, len(datastore.GetAllChangeTypes()))
+}
+
+func TestRegisterReplicationLatency(t *testing.T) {
+	t.Parallel()
+
+	registry := prometheus.NewRegistry()
+	_, err := RegisterReplicationLatency(gitalycfgprom.Config{
+		GRPCLatencyBuckets: prometheus.LinearBuckets(0, 2, 10),
+	}, registry)
+	require.NoError(t, err)
+
+	// We get the metric family, it should be instantiated with the 0 values
+	mfs, err := registry.Gather()
+	require.NoError(t, err)
+
+	// We only expect one metric family
+	require.Len(t, mfs, 1)
+
+	metricFamily := mfs[0]
+	require.Equal(t, "gitaly_praefect_replication_latency", metricFamily.GetName())
+	// The number of metrics should be equal to number of label values
+	require.Len(t, metricFamily.Metric, len(datastore.GetAllChangeTypes()))
+}