Add node gauge that keeps track of node status

A gauge is an easy way to visualize which nodes are primary and
which are secondary.

3 files changed, 43 insertions, 25 deletions

diff --git a/changelogs/unreleased/jc-add-node-gauge.yml b/changelogs/unreleased/jc-add-node-gauge.yml
new file mode 100644
index 000000000..98d500440
--- /dev/null
+++ b/changelogs/unreleased/jc-add-node-gauge.yml
@@ -0,0 +1,5 @@
+---
+title: Add node gauge that keeps track of node status
+merge_request: 1904
+author:
+type: changed
diff --git a/internal/praefect/metrics/prometheus.go b/internal/praefect/metrics/prometheus.go
index 2a8981750..12a7d7dfc 100644
--- a/internal/praefect/metrics/prometheus.go
+++ b/internal/praefect/metrics/prometheus.go
@@ -46,7 +46,7 @@ var PrimaryGauge = prometheus.NewGaugeVec(
 		Namespace: "gitaly",
 		Subsystem: "praefect",
 		Name:      "primaries",
-	}, []string{"storage"},
+	}, []string{"virtual_storage", "gitaly_storage"},
 )
 
 func init() {
diff --git a/internal/praefect/nodes/manager.go b/internal/praefect/nodes/manager.go
index d45affbfe..e9f2de58a 100644
--- a/internal/praefect/nodes/manager.go
+++ b/internal/praefect/nodes/manager.go
@@ -12,6 +12,7 @@ import (
 	"gitlab.com/gitlab-org/gitaly/client"
 	"gitlab.com/gitlab-org/gitaly/internal/praefect/config"
 	"gitlab.com/gitlab-org/gitaly/internal/praefect/grpc-proxy/proxy"
+	"gitlab.com/gitlab-org/gitaly/internal/praefect/metrics"
 	"gitlab.com/gitlab-org/gitaly/internal/praefect/models"
 	"google.golang.org/grpc"
 	healthpb "google.golang.org/grpc/health/grpc_health_v1"
@@ -65,6 +66,7 @@ func (s *shard) GetSecondaries() ([]Node, error) {
 
 // Mgr is a concrete type that adheres to the Manager interface
 type Mgr struct {
+	// shards is a map of shards keyed on virtual storage name
 	shards map[string]*shard
 	// staticShards never changes based on node health. It is a static set of shards that comes from the config's
 	// VirtualStorages
@@ -170,35 +172,46 @@ func (n *Mgr) GetShard(virtualStorageName string) (Shard, error) {
 	return shard, nil
 }
 
-func (n *Mgr) checkShards() {
-	for _, shard := range n.shards {
-		shard.primary.check()
-		for _, secondary := range shard.secondaries {
-			secondary.check()
+func checkShard(virtualStorage string, s *shard) {
+	defer func() {
+		metrics.PrimaryGauge.WithLabelValues(virtualStorage, s.primary.GetStorage()).Set(1)
+		for _, secondary := range s.secondaries {
+			metrics.PrimaryGauge.WithLabelValues(virtualStorage, secondary.GetStorage()).Set(0)
 		}
+	}()
 
-		if shard.primary.isHealthy() {
-			continue
-		}
+	s.primary.check()
+	for _, secondary := range s.secondaries {
+		secondary.check()
+	}
 
-		newPrimaryIndex := -1
-		for i, secondary := range shard.secondaries {
-			if secondary.isHealthy() {
-				newPrimaryIndex = i
-				break
-			}
-		}
+	if s.primary.isHealthy() {
+		return
+	}
 
-		if newPrimaryIndex < 0 {
-			// no healthy secondaries exist
-			continue
+	newPrimaryIndex := -1
+	for i, secondary := range s.secondaries {
+		if secondary.isHealthy() {
+			newPrimaryIndex = i
+			break
 		}
-		shard.m.Lock()
-		newPrimary := shard.secondaries[newPrimaryIndex]
-		shard.secondaries = append(shard.secondaries[:newPrimaryIndex], shard.secondaries[newPrimaryIndex+1:]...)
-		shard.secondaries = append(shard.secondaries, shard.primary)
-		shard.primary = newPrimary
-		shard.m.Unlock()
+	}
+
+	if newPrimaryIndex < 0 {
+		// no healthy secondaries exist
+		return
+	}
+	s.m.Lock()
+	newPrimary := s.secondaries[newPrimaryIndex]
+	s.secondaries = append(s.secondaries[:newPrimaryIndex], s.secondaries[newPrimaryIndex+1:]...)
+	s.secondaries = append(s.secondaries, s.primary)
+	s.primary = newPrimary
+	s.m.Unlock()
+}
+
+func (n *Mgr) checkShards() {
+	for virtualStorage, shard := range n.shards {
+		checkShard(virtualStorage, shard)
 	}
 }