Wire metadata verifier in Praefect's mainsmh-background-verifier

This commit wires the metadata verifier in Praefect's main so it can
actually be configured for use. It's default disabled still as it still
is missing some functionality that should be in place before generally
enabling it, for example tooling like metrics, integration in to the
'praefect metadata' tool and a background routine to release stale leases.

Changelog: added

4 files changed, 54 insertions, 14 deletions

diff --git a/cmd/praefect/main.go b/cmd/praefect/main.go
index 7f072b0f3..20f7b841f 100644
--- a/cmd/praefect/main.go
+++ b/cmd/praefect/main.go
@@ -349,6 +349,24 @@ func run(
 			rs,
 			conf.DefaultReplicationFactors(),
 		)
+
+		go func() {
+			if conf.BackgroundVerification.VerificationInterval <= 0 {
+				logger.Info("background verifier is disabled")
+				return
+			}
+
+			logger.WithField("config", conf.BackgroundVerification).Info("background verifier started")
+			if err := praefect.NewMetadataVerifier(
+				logger,
+				db,
+				nodeSet.Connections(),
+				hm,
+				conf.BackgroundVerification.VerificationInterval,
+			).Run(ctx, helper.NewTimerTicker(2*time.Second)); err != nil {
+				logger.WithError(err).Error("metadata verifier finished")
+			}
+		}()
 	} else {
 		if conf.Failover.Enabled {
 			logger.WithField("election_strategy", conf.Failover.ElectionStrategy).Warn(
diff --git a/internal/praefect/config/config.go b/internal/praefect/config/config.go
index 286577d6a..7c9a2791c 100644
--- a/internal/praefect/config/config.go
+++ b/internal/praefect/config/config.go
@@ -78,6 +78,18 @@ func (f Failover) ErrorThresholdsConfigured() (bool, error) {
 	return true, nil
 }
 
+// BackgroundVerification contains configuration options for the repository background verification.
+type BackgroundVerification struct {
+	// VerificationInterval determines the duration after a replica due for reverification.
+	// The feature is disabled if verification interval is 0 or below.
+	VerificationInterval time.Duration `toml:"verification_interval,omitempty"`
+}
+
+// DefaultBackgroundVerificationConfig returns the default background verification configuration.
+func DefaultBackgroundVerificationConfig() BackgroundVerification {
+	return BackgroundVerification{}
+}
+
 // Reconciliation contains reconciliation specific configuration options.
 type Reconciliation struct {
 	// SchedulingInterval the interval between each automatic reconciliation run. If set to 0,
@@ -112,17 +124,18 @@ func DefaultReplicationConfig() Replication {
 
 // Config is a container for everything found in the TOML config file
 type Config struct {
-	AllowLegacyElectors  bool              `toml:"i_understand_my_election_strategy_is_unsupported_and_will_be_removed_without_warning,omitempty"`
-	Reconciliation       Reconciliation    `toml:"reconciliation,omitempty"`
-	Replication          Replication       `toml:"replication,omitempty"`
-	ListenAddr           string            `toml:"listen_addr,omitempty"`
-	TLSListenAddr        string            `toml:"tls_listen_addr,omitempty"`
-	SocketPath           string            `toml:"socket_path,omitempty"`
-	VirtualStorages      []*VirtualStorage `toml:"virtual_storage,omitempty"`
-	Logging              log.Config        `toml:"logging,omitempty"`
-	Sentry               sentry.Config     `toml:"sentry,omitempty"`
-	PrometheusListenAddr string            `toml:"prometheus_listen_addr,omitempty"`
-	Prometheus           prometheus.Config `toml:"prometheus,omitempty"`
+	AllowLegacyElectors    bool                   `toml:"i_understand_my_election_strategy_is_unsupported_and_will_be_removed_without_warning,omitempty"`
+	BackgroundVerification BackgroundVerification `toml:"background_verification,omitempty"`
+	Reconciliation         Reconciliation         `toml:"reconciliation,omitempty"`
+	Replication            Replication            `toml:"replication,omitempty"`
+	ListenAddr             string                 `toml:"listen_addr,omitempty"`
+	TLSListenAddr          string                 `toml:"tls_listen_addr,omitempty"`
+	SocketPath             string                 `toml:"socket_path,omitempty"`
+	VirtualStorages        []*VirtualStorage      `toml:"virtual_storage,omitempty"`
+	Logging                log.Config             `toml:"logging,omitempty"`
+	Sentry                 sentry.Config          `toml:"sentry,omitempty"`
+	PrometheusListenAddr   string                 `toml:"prometheus_listen_addr,omitempty"`
+	Prometheus             prometheus.Config      `toml:"prometheus,omitempty"`
 	// PrometheusExcludeDatabaseFromDefaultMetrics excludes database-related metrics from the
 	// default metrics. If set to `false`, then database metrics will be available both via
 	// `/metrics` and `/db_metrics`. Otherwise, they will only be accessible via `/db_metrics`.
@@ -160,9 +173,10 @@ func FromFile(filePath string) (Config, error) {
 	}
 
 	conf := &Config{
-		Reconciliation: DefaultReconciliationConfig(),
-		Replication:    DefaultReplicationConfig(),
-		Prometheus:     prometheus.DefaultConfig(),
+		BackgroundVerification: DefaultBackgroundVerificationConfig(),
+		Reconciliation:         DefaultReconciliationConfig(),
+		Replication:            DefaultReplicationConfig(),
+		Prometheus:             prometheus.DefaultConfig(),
 		PrometheusExcludeDatabaseFromDefaultMetrics: true,
 		// Sets the default Failover, to be overwritten when deserializing the TOML
 		Failover:            Failover{Enabled: true, ElectionStrategy: ElectionStrategyPerRepository},
diff --git a/internal/praefect/config/config_test.go b/internal/praefect/config/config_test.go
index d3c8e3cfb..aa0d0b3c5 100644
--- a/internal/praefect/config/config_test.go
+++ b/internal/praefect/config/config_test.go
@@ -335,6 +335,9 @@ func TestConfigParsing(t *testing.T) {
 					RunInterval:         config.Duration(3 * time.Second),
 					RepositoriesInBatch: 10,
 				},
+				BackgroundVerification: BackgroundVerification{
+					VerificationInterval: 24 * time.Hour,
+				},
 			},
 		},
 		{
@@ -360,6 +363,7 @@ func TestConfigParsing(t *testing.T) {
 					RunInterval:         config.Duration(4 * time.Second),
 					RepositoriesInBatch: 11,
 				},
+				BackgroundVerification: DefaultBackgroundVerificationConfig(),
 			},
 		},
 		{
@@ -382,6 +386,7 @@ func TestConfigParsing(t *testing.T) {
 					RunInterval:         config.Duration(24 * time.Hour),
 					RepositoriesInBatch: 16,
 				},
+				BackgroundVerification: DefaultBackgroundVerificationConfig(),
 			},
 		},
 		{
diff --git a/internal/praefect/config/testdata/config.toml b/internal/praefect/config/testdata/config.toml
index 839b807b0..3f2a370e2 100644
--- a/internal/praefect/config/testdata/config.toml
+++ b/internal/praefect/config/testdata/config.toml
@@ -6,6 +6,9 @@ prometheus_listen_addr = ""
 memory_queue_enabled = true
 graceful_stop_timeout = "30s"
 
+[background_verification]
+verification_interval = "24h"
+
 [replication]
 batch_size = 1
 parallel_storage_processing_workers = 2