Configuration of gitaly nodes health checks

As we are moving towards enabling all feature flags and
configuration options for praefect and gitaly in tests by
default we require a configurable bootstrap and check intervals
for the health check request praefect executes on gitaly nodes.
This needed because of 'test-with-praefect' verification we
have where praefect is used in front of the gitaly server in
order to verify the integration is works properly.

With this settings we are able to minimize bootstrap interval
to reduce time needed to await until praefect would be ready
to serve requests. Also monitoring interval configuration allows
to reduce time required to get a gitaly node become unhealthy.

Part of: https://gitlab.com/gitlab-org/gitaly/-/issues/2702

4 files changed, 31 insertions, 8 deletions

diff --git a/cmd/praefect/main.go b/cmd/praefect/main.go
index 8bba748c8..8154a9792 100644
--- a/cmd/praefect/main.go
+++ b/cmd/praefect/main.go
@@ -240,7 +240,7 @@ func run(cfgs []starter.Config, conf config.Config) error {
 	if err != nil {
 		return err
 	}
-	nodeManager.Start(1*time.Second, 3*time.Second)
+	nodeManager.Start(conf.Failover.BootstrapInterval.Duration(), conf.Failover.MonitorInterval.Duration())
 
 	transactionCounterMetric, err := metrics.RegisterTransactionCounter()
 	if err != nil {
diff --git a/internal/praefect/config/config.go b/internal/praefect/config/config.go
index 2f0b81b59..7bb7c23d6 100644
--- a/internal/praefect/config/config.go
+++ b/internal/praefect/config/config.go
@@ -20,6 +20,12 @@ type Failover struct {
 	ErrorThresholdWindow     config.Duration `toml:"error_threshold_window"`
 	WriteErrorThresholdCount uint32          `toml:"write_error_threshold_count"`
 	ReadErrorThresholdCount  uint32          `toml:"read_error_threshold_count"`
+	// BootstrapInterval allows set a time duration that would be used on startup to make initial health check.
+	// The default value is 1s.
+	BootstrapInterval config.Duration `toml:"bootstrap_interval"`
+	// MonitorInterval allows set a time duration that would be used after bootstrap is completed to execute health checks.
+	// The default value is 3s.
+	MonitorInterval config.Duration `toml:"monitor_interval"`
 }
 
 const sqlFailoverValue = "sql"
@@ -60,14 +66,14 @@ func FromFile(filePath string) (Config, error) {
 		return Config{}, err
 	}
 
-	conf.setDefaults()
-
 	// TODO: Remove this after failover_enabled has moved under a separate failover section. This is for
 	// backwards compatibility only
 	if conf.FailoverEnabled {
 		conf.Failover.Enabled = true
 	}
 
+	conf.setDefaults()
+
 	return *conf, nil
 }
 
@@ -145,6 +151,16 @@ func (c *Config) setDefaults() {
 	if c.GracefulStopTimeout.Duration() == 0 {
 		c.GracefulStopTimeout = config.Duration(time.Minute)
 	}
+
+	if c.Failover.Enabled {
+		if c.Failover.BootstrapInterval.Duration() == 0 {
+			c.Failover.BootstrapInterval = config.Duration(time.Second)
+		}
+
+		if c.Failover.MonitorInterval.Duration() == 0 {
+			c.Failover.MonitorInterval = config.Duration(3 * time.Second)
+		}
+	}
 }
 
 // VirtualStorageNames returns names of all virtual storages configured.
diff --git a/internal/praefect/config/config_test.go b/internal/praefect/config/config_test.go
index dff855221..5948ab0cb 100644
--- a/internal/praefect/config/config_test.go
+++ b/internal/praefect/config/config_test.go
@@ -255,6 +255,8 @@ func TestConfigParsing(t *testing.T) {
 					ErrorThresholdWindow:     config.Duration(20 * time.Second),
 					WriteErrorThresholdCount: 1500,
 					ReadErrorThresholdCount:  100,
+					BootstrapInterval:        config.Duration(1 * time.Second),
+					MonitorInterval:          config.Duration(3 * time.Second),
 				},
 			},
 		},
@@ -264,8 +266,10 @@ func TestConfigParsing(t *testing.T) {
 			expected: Config{
 				GracefulStopTimeout: config.Duration(time.Minute),
 				Failover: Failover{
-					Enabled:          false,
-					ElectionStrategy: "local",
+					Enabled:           false,
+					ElectionStrategy:  "local",
+					BootstrapInterval: config.Duration(5 * time.Second),
+					MonitorInterval:   config.Duration(10 * time.Second),
 				},
 			},
 		},
@@ -275,8 +279,10 @@ func TestConfigParsing(t *testing.T) {
 			expected: Config{
 				GracefulStopTimeout: config.Duration(time.Minute),
 				Failover: Failover{
-					Enabled:          true,
-					ElectionStrategy: sqlFailoverValue,
+					Enabled:           true,
+					ElectionStrategy:  sqlFailoverValue,
+					BootstrapInterval: config.Duration(time.Second),
+					MonitorInterval:   config.Duration(3 * time.Second),
 				},
 			},
 		},
diff --git a/internal/praefect/config/testdata/config.overwritedefaults.toml b/internal/praefect/config/testdata/config.overwritedefaults.toml
index 39771fcd6..f58ad6206 100644
--- a/internal/praefect/config/testdata/config.overwritedefaults.toml
+++ b/internal/praefect/config/testdata/config.overwritedefaults.toml
@@ -2,4 +2,5 @@
 enabled = false
 election_strategy = "local"
 read_only_after_failover = false
-
+bootstrap_interval = "5s"
+monitor_interval = "10s"