Export unavailable repositories metric

The current read-only repository count metric describes unavailable
repositories rather than read-only repositories. We have to keep the
name for backwards compatibility as some alerting rules and dashboards
depend on it. To make it possible to migrate to a more accurate metric
later, this commit adds another metric on the side with more accurate
name and description.

2 files changed, 22 insertions, 6 deletions

diff --git a/internal/praefect/datastore/collector.go b/internal/praefect/datastore/collector.go
index 014def75a..dc9500b47 100644
--- a/internal/praefect/datastore/collector.go
+++ b/internal/praefect/datastore/collector.go
@@ -10,6 +10,9 @@ import (
 	"gitlab.com/gitlab-org/gitaly/v14/internal/praefect/datastore/glsql"
 )
 
+// This is kept for backwards compatibility as some alerting rules depend on this.
+// The unavailable repositories is a more accurate description for the metric and
+// is exported below so we can migrate to it.
 var descReadOnlyRepositories = prometheus.NewDesc(
 	"gitaly_praefect_read_only_repositories",
 	"Number of repositories in read-only mode within a virtual storage.",
@@ -17,6 +20,13 @@ var descReadOnlyRepositories = prometheus.NewDesc(
 	nil,
 )
 
+var descUnavailableRepositories = prometheus.NewDesc(
+	"gitaly_praefect_unavailable_repositories",
+	"Number of repositories that have no healthy, up to date replicas.",
+	[]string{"virtual_storage"},
+	nil,
+)
+
 // RepositoryStoreCollector collects metrics from the RepositoryStore.
 type RepositoryStoreCollector struct {
 	log             logrus.FieldLogger
@@ -38,14 +48,16 @@ func (c *RepositoryStoreCollector) Describe(ch chan<- *prometheus.Desc) {
 }
 
 func (c *RepositoryStoreCollector) Collect(ch chan<- prometheus.Metric) {
-	readOnlyCounts, err := c.queryMetrics(context.TODO())
+	unavailableCounts, err := c.queryMetrics(context.TODO())
 	if err != nil {
 		c.log.WithError(err).Error("failed collecting read-only repository count metric")
 		return
 	}
 
 	for _, vs := range c.virtualStorages {
-		ch <- prometheus.MustNewConstMetric(descReadOnlyRepositories, prometheus.GaugeValue, float64(readOnlyCounts[vs]), vs)
+		for _, desc := range []*prometheus.Desc{descReadOnlyRepositories, descUnavailableRepositories} {
+			ch <- prometheus.MustNewConstMetric(desc, prometheus.GaugeValue, float64(unavailableCounts[vs]), vs)
+		}
 	}
 }
 
@@ -68,7 +80,7 @@ GROUP BY virtual_storage
 	}
 	defer rows.Close()
 
-	vsReadOnly := make(map[string]int)
+	vsUnavailable := make(map[string]int)
 	for rows.Next() {
 		var vs string
 		var count int
@@ -77,8 +89,8 @@ GROUP BY virtual_storage
 			return nil, fmt.Errorf("scan: %w", err)
 		}
 
-		vsReadOnly[vs] = count
+		vsUnavailable[vs] = count
 	}
 
-	return vsReadOnly, rows.Err()
+	return vsUnavailable, rows.Err()
 }
diff --git a/internal/praefect/datastore/collector_test.go b/internal/praefect/datastore/collector_test.go
index 0b7fab508..f840a6749 100644
--- a/internal/praefect/datastore/collector_test.go
+++ b/internal/praefect/datastore/collector_test.go
@@ -179,7 +179,11 @@ func TestRepositoryStoreCollector(t *testing.T) {
 # TYPE gitaly_praefect_read_only_repositories gauge
 gitaly_praefect_read_only_repositories{virtual_storage="virtual-storage-1"} %d
 gitaly_praefect_read_only_repositories{virtual_storage="virtual-storage-2"} 0
-			`, tc.count))))
+# HELP gitaly_praefect_unavailable_repositories Number of repositories that have no healthy, up to date replicas.
+# TYPE gitaly_praefect_unavailable_repositories gauge
+gitaly_praefect_unavailable_repositories{virtual_storage="virtual-storage-1"} %d
+gitaly_praefect_unavailable_repositories{virtual_storage="virtual-storage-2"} 0
+			`, tc.count, tc.count))))
 		})
 	}
 }