Merge branch 'smh-verification-metrics' into 'master'

Instrument verification worker for observability

Closes #4097

See merge request gitlab-org/gitaly!4473

5 files changed, 383 insertions, 27 deletions

diff --git a/cmd/praefect/main.go b/cmd/praefect/main.go
index d33183438..ab28bdea0 100644
--- a/cmd/praefect/main.go
+++ b/cmd/praefect/main.go
@@ -359,6 +359,7 @@ func run(
 				hm,
 				conf.BackgroundVerification.VerificationInterval,
 			)
+			promreg.MustRegister(verifier)
 
 			go func() {
 				if err := verifier.Run(ctx, helper.NewTimerTicker(2*time.Second)); err != nil {
@@ -437,12 +438,20 @@ func run(
 	)
 	metricsCollectors = append(metricsCollectors, transactionManager, coordinator, repl)
 	if db != nil {
-		repositoryStoreCollector := datastore.NewRepositoryStoreCollector(
-			logger,
-			conf.VirtualStorageNames(),
-			db,
-			conf.Prometheus.ScrapeTimeout)
-		queueDepthCollector := datastore.NewQueueDepthCollector(logger, db, conf.Prometheus.ScrapeTimeout)
+		dbMetricCollectors := []prometheus.Collector{
+			datastore.NewRepositoryStoreCollector(logger, conf.VirtualStorageNames(), db, conf.Prometheus.ScrapeTimeout),
+			datastore.NewQueueDepthCollector(logger, db, conf.Prometheus.ScrapeTimeout),
+		}
+
+		if conf.BackgroundVerification.VerificationInterval > 0 {
+			dbMetricCollectors = append(dbMetricCollectors, datastore.NewVerificationQueueDepthCollector(
+				logger,
+				db,
+				conf.Prometheus.ScrapeTimeout,
+				conf.BackgroundVerification.VerificationInterval,
+				conf.StorageNames(),
+			))
+		}
 
 		// Eventually, database-related metrics will always be exported via a separate
 		// endpoint such that it's possible to set a different scraping interval and thus to
@@ -450,9 +459,9 @@ func run(
 		// standard and once for the database-specific endpoint. This is done to ensure a
 		// transitory period where deployments can be moved to the new endpoint without
 		// causing breakage if they still use the old endpoint.
-		dbPromRegistry.MustRegister(repositoryStoreCollector, queueDepthCollector)
+		dbPromRegistry.MustRegister(dbMetricCollectors...)
 		if !conf.PrometheusExcludeDatabaseFromDefaultMetrics {
-			promreg.MustRegister(repositoryStoreCollector, queueDepthCollector)
+			promreg.MustRegister(dbMetricCollectors...)
 		}
 	}
 	promreg.MustRegister(metricsCollectors...)
diff --git a/internal/praefect/datastore/collector.go b/internal/praefect/datastore/collector.go
index 0b93bd760..cd9bbfc28 100644
--- a/internal/praefect/datastore/collector.go
+++ b/internal/praefect/datastore/collector.go
@@ -178,3 +178,97 @@ GROUP BY job->>'virtual_storage', job->>'target_node_storage', state
 		q.log.WithError(err).Error("failed to iterate over rows for queue depth metrics")
 	}
 }
+
+const (
+	statusUnverified = "unverified"
+	statusExpired    = "expired"
+)
+
+// VerificationQueueDepthCollector collects the verification queue depth metric from the database.
+type VerificationQueueDepthCollector struct {
+	log                    logrus.FieldLogger
+	timeout                time.Duration
+	db                     glsql.Querier
+	verificationInterval   time.Duration
+	verificationQueueDepth *prometheus.GaugeVec
+}
+
+// NewVerificationQueueDepthCollector returns a new VerificationQueueDepthCollector
+func NewVerificationQueueDepthCollector(log logrus.FieldLogger, db glsql.Querier, timeout, verificationInterval time.Duration, configuredStorages map[string][]string) *VerificationQueueDepthCollector {
+	v := &VerificationQueueDepthCollector{
+		log:                  log.WithField("component", "verification_queue_depth_collector"),
+		timeout:              timeout,
+		db:                   db,
+		verificationInterval: verificationInterval,
+		verificationQueueDepth: prometheus.NewGaugeVec(prometheus.GaugeOpts{
+			Name: "gitaly_praefect_verification_queue_depth",
+			Help: "Number of replicas pending verification.",
+		}, []string{"virtual_storage", "storage", "status"}),
+	}
+
+	// pre-warm metrics to produce output even for storages which have an empty queue.
+	for virtualStorage, storages := range configuredStorages {
+		for _, storage := range storages {
+			for _, status := range []string{statusUnverified, statusExpired} {
+				v.verificationQueueDepth.WithLabelValues(virtualStorage, storage, status)
+			}
+		}
+	}
+
+	return v
+}
+
+// Describe describes the collected metrics to Prometheus.
+func (c *VerificationQueueDepthCollector) Describe(ch chan<- *prometheus.Desc) {
+	c.verificationQueueDepth.Describe(ch)
+}
+
+// Collect collects the verification queue depth metric from the database.
+func (c *VerificationQueueDepthCollector) Collect(ch chan<- prometheus.Metric) {
+	ctx, cancel := context.WithTimeout(context.TODO(), c.timeout)
+	defer cancel()
+
+	rows, err := c.db.QueryContext(ctx, `
+SELECT
+	repositories.virtual_storage,
+	storage,
+	COUNT(*) FILTER (WHERE verified_at IS NULL),
+	COUNT(*) FILTER (WHERE verified_at IS NOT NULL)
+FROM repositories
+JOIN storage_repositories USING (repository_id)
+WHERE verified_at IS NULL
+OR    verified_at < now() - $1 * '1 microsecond'::interval
+GROUP BY repositories.virtual_storage, storage
+`, c.verificationInterval.Microseconds())
+	if err != nil {
+		c.log.WithError(err).Error("failed to query verification queue depth metric")
+		return
+	}
+	defer rows.Close()
+
+	for rows.Next() {
+		var virtualStorage, storage string
+		var unverified, expired float64
+
+		if err := rows.Scan(&virtualStorage, &storage, &unverified, &expired); err != nil {
+			c.log.WithError(err).Error("failed to scan verification queue depth row")
+			return
+		}
+
+		for _, metric := range []struct {
+			status string
+			value  float64
+		}{
+			{status: statusUnverified, value: unverified},
+			{status: statusExpired, value: expired},
+		} {
+			c.verificationQueueDepth.WithLabelValues(virtualStorage, storage, metric.status).Set(metric.value)
+		}
+	}
+
+	if err := rows.Err(); err != nil {
+		c.log.WithError(err).Error("failed read verification queue depth rows")
+	}
+
+	c.verificationQueueDepth.Collect(ch)
+}
diff --git a/internal/praefect/datastore/collector_test.go b/internal/praefect/datastore/collector_test.go
index 3091c8321..c2e733268 100644
--- a/internal/praefect/datastore/collector_test.go
+++ b/internal/praefect/datastore/collector_test.go
@@ -324,3 +324,56 @@ gitaly_praefect_replication_queue_depth{state="ready",target_node="storage-1",vi
 gitaly_praefect_replication_queue_depth{state="ready",target_node="storage-4",virtual_storage="praefect-1"} %d
 `, 1, 1, readyJobs-1, readyJobs-1))))
 }
+
+func TestVerificationQueueDepthCollector(t *testing.T) {
+	ctx := testhelper.Context(t)
+
+	tx := testdb.New(t).Begin(t)
+	defer tx.Rollback(t)
+
+	rs := NewPostgresRepositoryStore(tx, nil)
+	require.NoError(t,
+		rs.CreateRepository(ctx, 1, "virtual-storage-1", "relative-path-1", "replica-path-1", "gitaly-1", []string{"gitaly-2", "gitaly-3"}, nil, true, false),
+	)
+	require.NoError(t,
+		rs.CreateRepository(ctx, 2, "virtual-storage-1", "relative-path-2", "replica-path-2", "gitaly-1", []string{"gitaly-2", "gitaly-3"}, nil, true, false),
+	)
+	require.NoError(t,
+		rs.CreateRepository(ctx, 3, "virtual-storage-2", "relative-path-1", "replica-path-3", "gitaly-1", []string{"gitaly-2", "gitaly-3"}, nil, true, false),
+	)
+
+	_, err := tx.ExecContext(ctx, `
+UPDATE storage_repositories
+SET verified_at = CASE
+	WHEN storage = 'gitaly-2' THEN now() - '30 seconds'::interval
+	ELSE now() - '30 seconds'::interval - '1 microsecond'::interval
+END
+WHERE virtual_storage = 'virtual-storage-1' AND storage != 'gitaly-1'
+	`)
+	require.NoError(t, err)
+
+	logger, hook := test.NewNullLogger()
+	require.NoError(t, testutil.CollectAndCompare(
+		NewVerificationQueueDepthCollector(logrus.NewEntry(logger), tx, time.Minute, 30*time.Second, map[string][]string{
+			"virtual-storage-1": {"gitaly-1", "gitaly-2", "gitaly-3"},
+			"virtual-storage-2": {"gitaly-1", "gitaly-2", "gitaly-3"},
+		}),
+		strings.NewReader(`
+# HELP gitaly_praefect_verification_queue_depth Number of replicas pending verification.
+# TYPE gitaly_praefect_verification_queue_depth gauge
+gitaly_praefect_verification_queue_depth{status="expired",storage="gitaly-1",virtual_storage="virtual-storage-1"} 0
+gitaly_praefect_verification_queue_depth{status="expired",storage="gitaly-1",virtual_storage="virtual-storage-2"} 0
+gitaly_praefect_verification_queue_depth{status="expired",storage="gitaly-2",virtual_storage="virtual-storage-1"} 0
+gitaly_praefect_verification_queue_depth{status="expired",storage="gitaly-2",virtual_storage="virtual-storage-2"} 0
+gitaly_praefect_verification_queue_depth{status="expired",storage="gitaly-3",virtual_storage="virtual-storage-1"} 2
+gitaly_praefect_verification_queue_depth{status="expired",storage="gitaly-3",virtual_storage="virtual-storage-2"} 0
+gitaly_praefect_verification_queue_depth{status="unverified",storage="gitaly-1",virtual_storage="virtual-storage-1"} 2
+gitaly_praefect_verification_queue_depth{status="unverified",storage="gitaly-1",virtual_storage="virtual-storage-2"} 1
+gitaly_praefect_verification_queue_depth{status="unverified",storage="gitaly-2",virtual_storage="virtual-storage-1"} 0
+gitaly_praefect_verification_queue_depth{status="unverified",storage="gitaly-2",virtual_storage="virtual-storage-2"} 1
+gitaly_praefect_verification_queue_depth{status="unverified",storage="gitaly-3",virtual_storage="virtual-storage-1"} 0
+gitaly_praefect_verification_queue_depth{status="unverified",storage="gitaly-3",virtual_storage="virtual-storage-2"} 1
+		`),
+	))
+	require.Empty(t, hook.AllEntries())
+}
diff --git a/internal/praefect/verifier.go b/internal/praefect/verifier.go
index 449e379ea..cf23ae7f4 100644
--- a/internal/praefect/verifier.go
+++ b/internal/praefect/verifier.go
@@ -7,6 +7,7 @@ import (
 	"sync"
 	"time"
 
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/sirupsen/logrus"
 	"gitlab.com/gitlab-org/gitaly/v14/internal/helper"
 	"gitlab.com/gitlab-org/gitaly/v14/internal/praefect/datastore/glsql"
@@ -27,8 +28,18 @@ type MetadataVerifier struct {
 	leaseDuration        time.Duration
 	healthChecker        HealthChecker
 	verificationInterval time.Duration
+
+	dequeuedJobsTotal        *prometheus.CounterVec
+	completedJobsTotal       *prometheus.CounterVec
+	staleLeasesReleasedTotal prometheus.Counter
 }
 
+const (
+	resultError   = "error"
+	resultInvalid = "invalid"
+	resultValid   = "valid"
+)
+
 // NewMetadataVerifier creates a new MetadataVerifier.
 func NewMetadataVerifier(
 	log logrus.FieldLogger,
@@ -37,7 +48,7 @@ func NewMetadataVerifier(
 	healthChecker HealthChecker,
 	verificationInterval time.Duration,
 ) *MetadataVerifier {
-	return &MetadataVerifier{
+	v := &MetadataVerifier{
 		log:                  log,
 		db:                   db,
 		conns:                conns,
@@ -45,7 +56,39 @@ func NewMetadataVerifier(
 		leaseDuration:        30 * time.Second,
 		healthChecker:        healthChecker,
 		verificationInterval: verificationInterval,
+		dequeuedJobsTotal: prometheus.NewCounterVec(
+			prometheus.CounterOpts{
+				Name: "gitaly_praefect_verification_jobs_dequeued_total",
+				Help: "Number of verification jobs dequeud.",
+			},
+			[]string{"virtual_storage", "storage"},
+		),
+		completedJobsTotal: prometheus.NewCounterVec(
+			prometheus.CounterOpts{
+				Name: "gitaly_praefect_verification_jobs_completed_total",
+				Help: "Number of verification jobs completed and their result",
+			},
+			[]string{"virtual_storage", "storage", "result"},
+		),
+		staleLeasesReleasedTotal: prometheus.NewCounter(
+			prometheus.CounterOpts{
+				Name: "gitaly_praefect_stale_verification_leases_released_total",
+				Help: "Number of stale verification leases released.",
+			},
+		),
 	}
+
+	// pre-warm the metrics so all labels are exported prior to their first observation
+	for virtualStorage, storages := range conns {
+		for storage := range storages {
+			v.dequeuedJobsTotal.WithLabelValues(virtualStorage, storage)
+			for _, result := range []string{resultError, resultInvalid, resultValid} {
+				v.completedJobsTotal.WithLabelValues(virtualStorage, storage, result)
+			}
+		}
+	}
+
+	return v
 }
 
 type verificationJob struct {
@@ -150,6 +193,7 @@ func (v *MetadataVerifier) releaseExpiredLeases(ctx context.Context) error {
 		}
 
 		if totalReleased > 0 {
+			v.staleLeasesReleasedTotal.Add(float64(totalReleased))
 			v.log.WithField("leases_released", released).Info("released stale verification leases")
 		}
 
@@ -179,6 +223,8 @@ func (v *MetadataVerifier) run(ctx context.Context) error {
 		go func() {
 			defer wg.Done()
 
+			v.dequeuedJobsTotal.WithLabelValues(job.virtualStorage, job.storage).Inc()
+
 			exists, err := v.verify(ctx, jobs[i])
 			results[i] = verificationResult{
 				job:    job,
@@ -190,7 +236,23 @@ func (v *MetadataVerifier) run(ctx context.Context) error {
 
 	wg.Wait()
 
-	return v.updateMetadata(ctx, results)
+	if err := v.updateMetadata(ctx, results); err != nil {
+		return fmt.Errorf("update metadata: %w", err)
+	}
+
+	for _, r := range results {
+		result := resultError
+		if r.error == nil {
+			result = resultInvalid
+			if r.exists {
+				result = resultValid
+			}
+		}
+
+		v.completedJobsTotal.WithLabelValues(r.job.virtualStorage, r.job.storage, result).Inc()
+	}
+
+	return nil
 }
 
 // logRecord is a helper type for gathering the removed replicas and logging them.
@@ -357,3 +419,15 @@ func (v *MetadataVerifier) verify(ctx context.Context, job verificationJob) (boo
 
 	return resp.Exists, nil
 }
+
+// Describe describes the collected metrics to Prometheus.
+func (v *MetadataVerifier) Describe(ch chan<- *prometheus.Desc) {
+	prometheus.DescribeByCollect(v, ch)
+}
+
+// Collect collects the metrics exposed from the MetadataVerifier.
+func (v *MetadataVerifier) Collect(ch chan<- prometheus.Metric) {
+	v.dequeuedJobsTotal.Collect(ch)
+	v.completedJobsTotal.Collect(ch)
+	v.staleLeasesReleasedTotal.Collect(ch)
+}
diff --git a/internal/praefect/verifier_test.go b/internal/praefect/verifier_test.go
index bb2a099b9..9c2ffda9b 100644
--- a/internal/praefect/verifier_test.go
+++ b/internal/praefect/verifier_test.go
@@ -5,9 +5,11 @@ import (
 	"errors"
 	"fmt"
 	"math/rand"
+	"strings"
 	"testing"
 	"time"
 
+	"github.com/prometheus/client_golang/prometheus/testutil"
 	"github.com/sirupsen/logrus"
 	"github.com/sirupsen/logrus/hooks/test"
 	"github.com/stretchr/testify/require"
@@ -73,16 +75,18 @@ func TestVerifier(t *testing.T) {
 	type step struct {
 		expectedRemovals logRecord
 		expectedErrors   map[string]map[string][]string
-		healthyStorages  StaticHealthChecker
 		expectedReplicas map[string]map[string][]string
 	}
 
 	for _, tc := range []struct {
-		desc            string
-		erroringGitalys map[string]bool
-		replicas        replicas
-		batchSize       int
-		steps           []step
+		desc               string
+		erroringGitalys    map[string]bool
+		replicas           replicas
+		healthyStorages    StaticHealthChecker
+		batchSize          int
+		steps              []step
+		dequeuedJobsTotal  map[string]map[string]int
+		completedJobsTotal map[string]map[string]map[string]int
 	}{
 		{
 			desc: "all replicas exist",
@@ -104,6 +108,18 @@ func TestVerifier(t *testing.T) {
 					},
 				},
 			},
+			dequeuedJobsTotal: map[string]map[string]int{
+				"virtual-storage": {
+					gitaly1: 1,
+					gitaly3: 1,
+				},
+			},
+			completedJobsTotal: map[string]map[string]map[string]int{
+				"virtual-storage": {
+					gitaly1: {"valid": 1},
+					gitaly3: {"valid": 1},
+				},
+			},
 		},
 		{
 			desc: "recently verified replicas are not picked",
@@ -137,9 +153,9 @@ func TestVerifier(t *testing.T) {
 					},
 				},
 			},
+			healthyStorages: StaticHealthChecker{"virtual-storage": {gitaly1, gitaly2}},
 			steps: []step{
 				{
-					healthyStorages: StaticHealthChecker{"virtual-storage": {gitaly1, gitaly2}},
 					expectedReplicas: map[string]map[string][]string{
 						"virtual-storage": {
 							"repository-1": {gitaly1, gitaly2, gitaly3},
@@ -147,6 +163,18 @@ func TestVerifier(t *testing.T) {
 					},
 				},
 			},
+			dequeuedJobsTotal: map[string]map[string]int{
+				"virtual-storage": {
+					gitaly1: 1,
+					gitaly2: 1,
+				},
+			},
+			completedJobsTotal: map[string]map[string]map[string]int{
+				"virtual-storage": {
+					gitaly1: {"valid": 1},
+					gitaly2: {"valid": 1},
+				},
+			},
 		},
 		{
 			desc: "metadata not deleted for replicas which errored on verification",
@@ -174,6 +202,20 @@ func TestVerifier(t *testing.T) {
 					},
 				},
 			},
+			dequeuedJobsTotal: map[string]map[string]int{
+				"virtual-storage": {
+					gitaly1: 1,
+					gitaly2: 1,
+					gitaly3: 1,
+				},
+			},
+			completedJobsTotal: map[string]map[string]map[string]int{
+				"virtual-storage": {
+					gitaly1: {"valid": 1},
+					gitaly2: {"valid": 1},
+					gitaly3: {"error": 1},
+				},
+			},
 		},
 		{
 			desc: "replicas with leases acquired are not picked",
@@ -221,6 +263,20 @@ func TestVerifier(t *testing.T) {
 					},
 				},
 			},
+			dequeuedJobsTotal: map[string]map[string]int{
+				"virtual-storage": {
+					gitaly1: 1,
+					gitaly2: 1,
+					gitaly3: 1,
+				},
+			},
+			completedJobsTotal: map[string]map[string]map[string]int{
+				"virtual-storage": {
+					gitaly1: {"valid": 1},
+					gitaly2: {"invalid": 1},
+					gitaly3: {"invalid": 1},
+				},
+			},
 		},
 		{
 			desc: "verification time is updated when repository exists",
@@ -267,6 +323,20 @@ func TestVerifier(t *testing.T) {
 					},
 				},
 			},
+			dequeuedJobsTotal: map[string]map[string]int{
+				"virtual-storage": {
+					gitaly1: 1,
+					gitaly2: 1,
+					gitaly3: 1,
+				},
+			},
+			completedJobsTotal: map[string]map[string]map[string]int{
+				"virtual-storage": {
+					gitaly1: {"valid": 1},
+					gitaly2: {"invalid": 1},
+					gitaly3: {"invalid": 1},
+				},
+			},
 		},
 		{
 			desc: "all replicas are lost",
@@ -316,6 +386,20 @@ func TestVerifier(t *testing.T) {
 					expectedReplicas: map[string]map[string][]string{},
 				},
 			},
+			dequeuedJobsTotal: map[string]map[string]int{
+				"virtual-storage": {
+					gitaly1: 1,
+					gitaly2: 1,
+					gitaly3: 1,
+				},
+			},
+			completedJobsTotal: map[string]map[string]map[string]int{
+				"virtual-storage": {
+					gitaly1: {"invalid": 1},
+					gitaly2: {"invalid": 1},
+					gitaly3: {"invalid": 1},
+				},
+			},
 		},
 	} {
 		t.Run(tc.desc, func(t *testing.T) {
@@ -490,18 +574,20 @@ func TestVerifier(t *testing.T) {
 			).Scan(&lockedRows))
 			require.Equal(t, 3, lockedRows)
 
-			for _, step := range tc.steps {
-				logger, hook := test.NewNullLogger()
+			logger, hook := test.NewNullLogger()
 
-				healthyStorages := StaticHealthChecker{"virtual-storage": []string{gitaly1, gitaly2, gitaly3}}
-				if step.healthyStorages != nil {
-					healthyStorages = step.healthyStorages
-				}
+			healthyStorages := StaticHealthChecker{"virtual-storage": []string{gitaly1, gitaly2, gitaly3}}
+			if tc.healthyStorages != nil {
+				healthyStorages = tc.healthyStorages
+			}
 
-				verifier := NewMetadataVerifier(logger, db, conns, healthyStorages, 24*7*time.Hour)
-				if tc.batchSize > 0 {
-					verifier.batchSize = tc.batchSize
-				}
+			verifier := NewMetadataVerifier(logger, db, conns, healthyStorages, 24*7*time.Hour)
+			if tc.batchSize > 0 {
+				verifier.batchSize = tc.batchSize
+			}
+
+			for _, step := range tc.steps {
+				hook.Reset()
 
 				runCtx, cancelRun := context.WithCancel(ctx)
 				err = verifier.Run(runCtx, helper.NewCountTicker(1, cancelRun))
@@ -558,6 +644,41 @@ func TestVerifier(t *testing.T) {
 				// Ensure all the metadata still contains the expected replicas
 				require.Equal(t, step.expectedReplicas, getAllReplicas(ctx, t, db))
 			}
+
+			require.NoError(t, testutil.CollectAndCompare(verifier, strings.NewReader(fmt.Sprintf(`
+# HELP gitaly_praefect_stale_verification_leases_released_total Number of stale verification leases released.
+# TYPE gitaly_praefect_stale_verification_leases_released_total counter
+gitaly_praefect_stale_verification_leases_released_total 0
+# HELP gitaly_praefect_verification_jobs_completed_total Number of verification jobs completed and their result
+# TYPE gitaly_praefect_verification_jobs_completed_total counter
+gitaly_praefect_verification_jobs_completed_total{result="error",storage="gitaly-0",virtual_storage="virtual-storage"} %d
+gitaly_praefect_verification_jobs_completed_total{result="error",storage="gitaly-1",virtual_storage="virtual-storage"} %d
+gitaly_praefect_verification_jobs_completed_total{result="error",storage="gitaly-2",virtual_storage="virtual-storage"} %d
+gitaly_praefect_verification_jobs_completed_total{result="invalid",storage="gitaly-0",virtual_storage="virtual-storage"} %d
+gitaly_praefect_verification_jobs_completed_total{result="invalid",storage="gitaly-1",virtual_storage="virtual-storage"} %d
+gitaly_praefect_verification_jobs_completed_total{result="invalid",storage="gitaly-2",virtual_storage="virtual-storage"} %d
+gitaly_praefect_verification_jobs_completed_total{result="valid",storage="gitaly-0",virtual_storage="virtual-storage"} %d
+gitaly_praefect_verification_jobs_completed_total{result="valid",storage="gitaly-1",virtual_storage="virtual-storage"} %d
+gitaly_praefect_verification_jobs_completed_total{result="valid",storage="gitaly-2",virtual_storage="virtual-storage"} %d
+# HELP gitaly_praefect_verification_jobs_dequeued_total Number of verification jobs dequeud.
+# TYPE gitaly_praefect_verification_jobs_dequeued_total counter
+gitaly_praefect_verification_jobs_dequeued_total{storage="gitaly-0",virtual_storage="virtual-storage"} %d
+gitaly_praefect_verification_jobs_dequeued_total{storage="gitaly-1",virtual_storage="virtual-storage"} %d
+gitaly_praefect_verification_jobs_dequeued_total{storage="gitaly-2",virtual_storage="virtual-storage"} %d
+			`,
+				tc.completedJobsTotal["virtual-storage"][gitaly1][resultError],
+				tc.completedJobsTotal["virtual-storage"][gitaly2][resultError],
+				tc.completedJobsTotal["virtual-storage"][gitaly3][resultError],
+				tc.completedJobsTotal["virtual-storage"][gitaly1][resultInvalid],
+				tc.completedJobsTotal["virtual-storage"][gitaly2][resultInvalid],
+				tc.completedJobsTotal["virtual-storage"][gitaly3][resultInvalid],
+				tc.completedJobsTotal["virtual-storage"][gitaly1][resultValid],
+				tc.completedJobsTotal["virtual-storage"][gitaly2][resultValid],
+				tc.completedJobsTotal["virtual-storage"][gitaly3][resultValid],
+				tc.dequeuedJobsTotal["virtual-storage"][gitaly1],
+				tc.dequeuedJobsTotal["virtual-storage"][gitaly2],
+				tc.dequeuedJobsTotal["virtual-storage"][gitaly3],
+			))))
 		})
 	}
 }
@@ -672,4 +793,9 @@ func TestVerifier_runExpiredLeaseReleaser(t *testing.T) {
 	require.Equal(t, map[string]map[string]map[string]struct{}{
 		"virtual-storage-1": {"relative-path-1": {"expired-lease-1": {}, "expired-lease-2": {}, "expired-lease-3": {}}},
 	}, actualReleased)
+	require.NoError(t, testutil.CollectAndCompare(verifier, strings.NewReader(`
+# HELP gitaly_praefect_stale_verification_leases_released_total Number of stale verification leases released.
+# TYPE gitaly_praefect_stale_verification_leases_released_total counter
+gitaly_praefect_stale_verification_leases_released_total 3
+	`)))
 }