praefect: Check of the service readiness with RPC callps-praefect-readiness-check

For the praefect binary we have a sub-command to verify
if praefect service can operate without issues. The
verification process checks if migrations were applied,
if gitaly nodes are reachable, if the clock synced, etc.
This check can be done only when you have direct access
to the binary. With introducing of ReadinessCheck RPC
we now can run the same verification process mentioned
above by issuing an RPC call.
The new RPC will be part of the gitlab:gitaly:check task.
It is noop for the gitaly service as of now.

Part of: https://gitlab.com/gitlab-org/gitlab/-/issues/348174

4 files changed, 191 insertions, 0 deletions

diff --git a/internal/gitaly/service/server/readiness.go b/internal/gitaly/service/server/readiness.go
new file mode 100644
index 000000000..59708e2ed
--- /dev/null
+++ b/internal/gitaly/service/server/readiness.go
@@ -0,0 +1,13 @@
+package server
+
+import (
+	"context"
+
+	"gitlab.com/gitlab-org/gitaly/v15/proto/go/gitalypb"
+)
+
+// ReadinessCheck is a stub that does nothing but exists to support single interface for gitaly
+// and praefect. The praefect service requires this method.
+func (s *server) ReadinessCheck(context.Context, *gitalypb.ReadinessCheckRequest) (*gitalypb.ReadinessCheckResponse, error) {
+	return &gitalypb.ReadinessCheckResponse{Result: &gitalypb.ReadinessCheckResponse_OkResponse{}}, nil
+}
diff --git a/internal/praefect/service/server/readiness.go b/internal/praefect/service/server/readiness.go
new file mode 100644
index 000000000..39f1392e8
--- /dev/null
+++ b/internal/praefect/service/server/readiness.go
@@ -0,0 +1,53 @@
+package server
+
+import (
+	"context"
+	"io"
+	"sort"
+
+	"gitlab.com/gitlab-org/gitaly/v15/proto/go/gitalypb"
+)
+
+// ReadinessCheck runs the set of the checks to make sure service is in operational state.
+func (s *Server) ReadinessCheck(ctx context.Context, req *gitalypb.ReadinessCheckRequest) (*gitalypb.ReadinessCheckResponse, error) {
+	checkCtx := ctx
+	checkCancel := func() {}
+	timeout := req.GetTimeout().AsDuration()
+	if req.GetTimeout().IsValid() && timeout > 0 {
+		checkCtx, checkCancel = context.WithTimeout(ctx, timeout)
+	}
+	defer checkCancel()
+
+	results := make(chan *gitalypb.ReadinessCheckResponse_Failure_Response, len(s.checks))
+	for _, newCheck := range s.checks {
+		check := newCheck(s.conf, io.Discard, true)
+		go func() {
+			if err := check.Run(checkCtx); err != nil {
+				results <- &gitalypb.ReadinessCheckResponse_Failure_Response{
+					Name:         check.Name,
+					ErrorMessage: err.Error(),
+				}
+			} else {
+				results <- nil
+			}
+		}()
+	}
+
+	var failedChecks []*gitalypb.ReadinessCheckResponse_Failure_Response
+	for i := 0; i < cap(results); i++ {
+		if result := <-results; result != nil {
+			failedChecks = append(failedChecks, result)
+		}
+	}
+
+	if len(failedChecks) > 0 {
+		sort.Slice(failedChecks, func(i, j int) bool { return failedChecks[i].Name < failedChecks[j].Name })
+		return &gitalypb.ReadinessCheckResponse{Result: &gitalypb.ReadinessCheckResponse_FailureResponse{
+			FailureResponse: &gitalypb.ReadinessCheckResponse_Failure{
+				FailedChecks: failedChecks,
+			},
+		}}, nil
+	}
+
+	return &gitalypb.ReadinessCheckResponse{Result: &gitalypb.ReadinessCheckResponse_OkResponse{}}, nil
+}
diff --git a/internal/praefect/service/server/readiness_test.go b/internal/praefect/service/server/readiness_test.go
new file mode 100644
index 000000000..d53deb413
--- /dev/null
+++ b/internal/praefect/service/server/readiness_test.go
@@ -0,0 +1,114 @@
+package server_test
+
+import (
+	"context"
+	"io"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"gitlab.com/gitlab-org/gitaly/v15/internal/gitaly/service/setup"
+	"gitlab.com/gitlab-org/gitaly/v15/internal/praefect"
+	"gitlab.com/gitlab-org/gitaly/v15/internal/praefect/config"
+	"gitlab.com/gitlab-org/gitaly/v15/internal/praefect/service"
+	"gitlab.com/gitlab-org/gitaly/v15/internal/testhelper"
+	"gitlab.com/gitlab-org/gitaly/v15/internal/testhelper/testcfg"
+	"gitlab.com/gitlab-org/gitaly/v15/internal/testhelper/testserver"
+	"gitlab.com/gitlab-org/gitaly/v15/proto/go/gitalypb"
+	"google.golang.org/protobuf/types/known/durationpb"
+)
+
+func TestServer_ReadinessCheck(t *testing.T) {
+	t.Parallel()
+	stubCheck := func(t *testing.T, triggered chan string, name string) *service.Check {
+		return &service.Check{
+			Name: name,
+			Run: func(ctx context.Context) error {
+				_, ok := ctx.Deadline()
+				assert.True(t, ok, "the deadline should be set as we provide timeout")
+				triggered <- name
+				return nil
+			},
+		}
+	}
+
+	const gitalyStorageName = "praefect-internal-0"
+	gitalyCfg := testcfg.Build(t, testcfg.WithStorages(gitalyStorageName))
+	gitalyAddr := testserver.RunGitalyServer(t, gitalyCfg, nil, setup.RegisterAll, testserver.WithDisablePraefect())
+
+	praefectConf := config.Config{
+		SocketPath: testhelper.GetTemporaryGitalySocketFileName(t),
+		VirtualStorages: []*config.VirtualStorage{
+			{
+				Name: "default",
+				Nodes: []*config.Node{
+					{
+						Storage: gitalyStorageName,
+						Address: gitalyAddr,
+					},
+				},
+			},
+		},
+	}
+	ctx := testhelper.Context(t)
+	triggered := make(chan string, 2)
+	grpcPraefectConn, _, cleanup := praefect.RunPraefectServer(t, ctx, praefectConf, praefect.BuildOptions{
+		WithChecks: []service.CheckFunc{
+			func(conf config.Config, w io.Writer, quiet bool) *service.Check {
+				return stubCheck(t, triggered, "1")
+			},
+			func(conf config.Config, w io.Writer, quiet bool) *service.Check {
+				return stubCheck(t, triggered, "2")
+			},
+		},
+	})
+	t.Cleanup(cleanup)
+	serverClient := gitalypb.NewServerServiceClient(grpcPraefectConn)
+	resp, err := serverClient.ReadinessCheck(ctx, &gitalypb.ReadinessCheckRequest{Timeout: durationpb.New(time.Second)})
+	require.NoError(t, err)
+	assert.NotNil(t, resp.GetOkResponse())
+	if !assert.Nil(t, resp.GetFailureResponse()) {
+		for _, failure := range resp.GetFailureResponse().GetFailedChecks() {
+			assert.Failf(t, "failed check", "%s: %s", failure.Name, failure.ErrorMessage)
+		}
+	}
+	names := make([]string, 0, cap(triggered))
+	for i := 0; i < cap(triggered); i++ {
+		name := <-triggered
+		names = append(names, name)
+	}
+	require.ElementsMatch(t, []string{"1", "2"}, names, "both tasks should be triggered for an execution")
+}
+
+func TestServer_ReadinessCheck_unreachableGitaly(t *testing.T) {
+	t.Parallel()
+	praefectConf := config.Config{
+		SocketPath: testhelper.GetTemporaryGitalySocketFileName(t),
+		VirtualStorages: []*config.VirtualStorage{
+			{
+				Name: "default",
+				Nodes: []*config.Node{
+					{
+						Storage: "praefect-internal-0",
+						Address: "tcp://non-existing:42",
+					},
+				},
+			},
+		},
+	}
+	ctx := testhelper.Context(t)
+	grpcConn, _, cleanup := praefect.RunPraefectServer(t, ctx, praefectConf, praefect.BuildOptions{})
+	t.Cleanup(cleanup)
+	serverClient := gitalypb.NewServerServiceClient(grpcConn)
+	resp, err := serverClient.ReadinessCheck(ctx, &gitalypb.ReadinessCheckRequest{Timeout: durationpb.New(time.Nanosecond)})
+	require.NoError(t, err)
+	require.Nil(t, resp.GetOkResponse())
+	require.NotNil(t, resp.GetFailureResponse())
+	require.Len(t, resp.GetFailureResponse().FailedChecks, 5)
+	require.Equal(t, "clock synchronization", resp.GetFailureResponse().FailedChecks[0].Name)
+	require.Equal(t, "database read/write", resp.GetFailureResponse().FailedChecks[1].Name)
+	require.Equal(t, "gitaly node connectivity & disk access", resp.GetFailureResponse().FailedChecks[2].Name)
+	require.Equal(t, "praefect migrations", resp.GetFailureResponse().FailedChecks[3].Name)
+	require.Equal(t, "unavailable repositories", resp.GetFailureResponse().FailedChecks[4].Name)
+}
diff --git a/internal/praefect/service/server/testhelper_test.go b/internal/praefect/service/server/testhelper_test.go
new file mode 100644
index 000000000..39eb55aaf
--- /dev/null
+++ b/internal/praefect/service/server/testhelper_test.go
@@ -0,0 +1,11 @@
+package server_test
+
+import (
+	"testing"
+
+	"gitlab.com/gitlab-org/gitaly/v15/internal/testhelper"
+)
+
+func TestMain(m *testing.M) {
+	testhelper.Run(m)
+}