Adding custom healthcheck servicejc-custom-health-check

12 files changed, 566 insertions, 9 deletions

diff --git a/internal/middleware/errorhandler/errorhandler.go b/internal/middleware/errorhandler/errorhandler.go
new file mode 100644
index 000000000..ea00510d0
--- /dev/null
+++ b/internal/middleware/errorhandler/errorhandler.go
@@ -0,0 +1,98 @@
+package errorhandler
+
+import (
+	"context"
+	"fmt"
+	"sync"
+	"time"
+
+	"gitlab.com/gitlab-org/gitaly/internal/praefect/protoregistry"
+
+	"google.golang.org/grpc"
+)
+
+func UnaryErrorHandler(errorTracker *Errors, registry *protoregistry.Registry) grpc.UnaryServerInterceptor {
+	return func(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) {
+		resp, err := handler(ctx, req)
+		if err != nil {
+			mi, lookupErr := registry.LookupMethod(info.FullMethod)
+			if err != nil {
+				return resp, fmt.Errorf("error when looking up method :%w %w", err, lookupErr)
+			}
+			switch mi.Operation {
+			case protoregistry.OpAccessor:
+				errorTracker.IncrReadErr()
+			case protoregistry.OpMutator:
+				errorTracker.IncrWriteErr()
+			}
+		}
+		return resp, err
+	}
+}
+
+func StreamErrorHandler(errorTracker *Errors, registry *protoregistry.Registry) grpc.StreamServerInterceptor {
+	return func(srv interface{}, stream grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) error {
+		err := handler(srv, stream)
+		if err != nil {
+			mi, lookupErr := registry.LookupMethod(info.FullMethod)
+			if err != nil {
+				return fmt.Errorf("error when looking up method :%w %w", err, lookupErr)
+			}
+			switch mi.Operation {
+			case protoregistry.OpAccessor:
+				errorTracker.IncrReadErr()
+			case protoregistry.OpMutator:
+				errorTracker.IncrWriteErr()
+			}
+		}
+
+		return err
+	}
+}
+
+type Errors struct {
+	m                       sync.RWMutex
+	readErrors, writeErrors int64
+}
+
+func (e *Errors) IncrReadErr() {
+	e.m.Lock()
+	defer e.m.Unlock()
+
+	e.readErrors++
+}
+
+func (e *Errors) IncrWriteErr() {
+	e.m.Lock()
+	defer e.m.Unlock()
+
+	e.writeErrors++
+}
+
+func (e *Errors) ReadErrs() int64 {
+	e.m.RLock()
+	defer e.m.RUnlock()
+
+	return e.readErrors
+}
+
+func (e *Errors) WriteErrs() int64 {
+	e.m.RLock()
+	defer e.m.RUnlock()
+
+	return e.writeErrors
+}
+
+func (e *Errors) PeriodicallyClear() {
+	ticker := time.NewTicker(10 * time.Second)
+	for {
+		<-ticker.C
+	}
+}
+
+func (e *Errors) clear() {
+	e.m.Lock()
+	defer e.m.Unlock()
+	e.readErrors = 0
+	e.writeErrors = 0
+}
diff --git a/internal/praefect/nodes/manager.go b/internal/praefect/nodes/manager.go
index 5d25551ba..27c3068df 100644
--- a/internal/praefect/nodes/manager.go
+++ b/internal/praefect/nodes/manager.go
@@ -6,6 +6,8 @@ import (
 	"errors"
 	"time"
 
+	"gitlab.com/gitlab-org/gitaly/proto/go/gitalypb"
+
 	grpc_middleware "github.com/grpc-ecosystem/go-grpc-middleware"
 	grpc_prometheus "github.com/grpc-ecosystem/go-grpc-prometheus"
 	"github.com/sirupsen/logrus"
@@ -19,7 +21,6 @@ import (
 	correlation "gitlab.com/gitlab-org/labkit/correlation/grpc"
 	grpctracing "gitlab.com/gitlab-org/labkit/tracing/grpc"
 	"google.golang.org/grpc"
-	healthpb "google.golang.org/grpc/health/grpc_health_v1"
 )
 
 // Shard is a primary with a set of secondaries
@@ -211,16 +212,16 @@ func (n *nodeStatus) GetConnection() *grpc.ClientConn {
 const checkTimeout = 1 * time.Second
 
 func (n *nodeStatus) check(ctx context.Context) (bool, error) {
-	client := healthpb.NewHealthClient(n.ClientConn)
+	client := gitalypb.NewHealthClient(n.ClientConn)
 	ctx, cancel := context.WithTimeout(ctx, checkTimeout)
 	defer cancel()
 	status := false
 
 	start := time.Now()
-	resp, err := client.Check(ctx, &healthpb.HealthCheckRequest{Service: ""})
+	resp, err := client.Check(ctx, &gitalypb.HealthCheckRequest{Service: ""})
 	n.latencyHist.WithLabelValues(n.Storage).Observe(time.Since(start).Seconds())
 
-	if err == nil && resp.Status == healthpb.HealthCheckResponse_SERVING {
+	if err == nil && resp.Status == gitalypb.HealthCheckResponse_SERVING {
 		status = true
 	} else {
 		n.log.WithError(err).WithFields(logrus.Fields{
diff --git a/internal/server/server.go b/internal/server/server.go
index 2240a4442..d27746837 100644
--- a/internal/server/server.go
+++ b/internal/server/server.go
@@ -5,6 +5,8 @@ import (
 	"crypto/tls"
 	"os"
 
+	"gitlab.com/gitlab-org/gitaly/internal/middleware/errorhandler"
+
 	grpc_middleware "github.com/grpc-ecosystem/go-grpc-middleware"
 	grpc_logrus "github.com/grpc-ecosystem/go-grpc-middleware/logging/logrus"
 	grpc_ctxtags "github.com/grpc-ecosystem/go-grpc-middleware/tags"
@@ -75,6 +77,9 @@ func createNewServer(rubyServer *rubyserver.Server, cfg config.Cfg, secure bool)
 	}
 
 	lh := limithandler.New(concurrencyKeyFn)
+	var errorTracker errorhandler.Errors
+
+	registry := protoregistry.New()
 
 	opts := []grpc.ServerOption{
 		grpc.StreamInterceptor(grpc_middleware.ChainStreamServer(
@@ -89,6 +94,7 @@ func createNewServer(rubyServer *rubyserver.Server, cfg config.Cfg, secure bool)
 			auth.StreamServerInterceptor(config.Config.Auth),
 			grpctracing.StreamServerTracingInterceptor(),
 			cache.StreamInvalidator(diskcache.LeaseKeyer{}, protoregistry.GitalyProtoPreregistered),
+			errorhandler.StreamErrorHandler(&errorTracker, registry),
 			// Panic handler should remain last so that application panics will be
 			// converted to errors and logged
 			panichandler.StreamPanicHandler,
@@ -107,6 +113,7 @@ func createNewServer(rubyServer *rubyserver.Server, cfg config.Cfg, secure bool)
 			cache.UnaryInvalidator(diskcache.LeaseKeyer{}, protoregistry.GitalyProtoPreregistered),
 			// Panic handler should remain last so that application panics will be
 			// converted to errors and logged
+			errorhandler.UnaryErrorHandler(&errorTracker, registry),
 			panichandler.UnaryPanicHandler,
 		)),
 	}
@@ -123,7 +130,7 @@ func createNewServer(rubyServer *rubyserver.Server, cfg config.Cfg, secure bool)
 
 	server := grpc.NewServer(opts...)
 
-	service.RegisterAll(server, cfg, rubyServer)
+	service.RegisterAll(server, cfg, &errorTracker, rubyServer)
 	reflection.Register(server)
 
 	grpc_prometheus.Register(server)
diff --git a/internal/service/health/check.go b/internal/service/health/check.go
new file mode 100644
index 000000000..9919d6df1
--- /dev/null
+++ b/internal/service/health/check.go
@@ -0,0 +1,42 @@
+package health
+
+import (
+	"context"
+	"time"
+
+	"github.com/grpc-ecosystem/go-grpc-middleware/logging/logrus/ctxlogrus"
+
+	"gitlab.com/gitlab-org/gitaly/proto/go/gitalypb"
+)
+
+func (s *server) Check(ctx context.Context, in *gitalypb.HealthCheckRequest) (*gitalypb.HealthCheckResponse, error) {
+	if s.errorTracker.ReadErrs() > 1000 || s.errorTracker.WriteErrs() > 100 {
+		ctxlogrus.Extract(ctx).Info("I'm not healthy 😵")
+		return &gitalypb.HealthCheckResponse{Status: gitalypb.HealthCheckResponse_NOT_SERVING}, nil
+	}
+	ctxlogrus.Extract(ctx).Info("I'm healthy 😁")
+	return &gitalypb.HealthCheckResponse{Status: gitalypb.HealthCheckResponse_SERVING}, nil
+}
+
+func (s *server) Watch(in *gitalypb.HealthCheckRequest, stream gitalypb.Health_WatchServer) error {
+	ticker := time.NewTicker(10 * time.Second)
+
+	for {
+		select {
+		case <-stream.Context().Done():
+			return nil
+		case <-ticker.C:
+			if s.errorTracker.ReadErrs() > 1000 || s.errorTracker.WriteErrs() > 100 {
+				if err := stream.Send(&gitalypb.HealthCheckResponse{Status: gitalypb.HealthCheckResponse_NOT_SERVING}); err != nil {
+					return err
+					continue
+				}
+				if err := stream.Send(&gitalypb.HealthCheckResponse{Status: gitalypb.HealthCheckResponse_SERVING}); err != nil {
+					return err
+				}
+			}
+		}
+	}
+
+	return nil
+}
diff --git a/internal/service/health/server.go b/internal/service/health/server.go
new file mode 100644
index 000000000..bc7d127ff
--- /dev/null
+++ b/internal/service/health/server.go
@@ -0,0 +1,15 @@
+package health
+
+import (
+	"gitlab.com/gitlab-org/gitaly/internal/middleware/errorhandler"
+	"gitlab.com/gitlab-org/gitaly/proto/go/gitalypb"
+)
+
+type server struct {
+	errorTracker *errorhandler.Errors
+}
+
+// NewServer creates a new instance of a gRPC namespace server
+func NewServer(errorTracker *errorhandler.Errors) gitalypb.HealthServer {
+	return &server{errorTracker: errorTracker}
+}
diff --git a/internal/service/register.go b/internal/service/register.go
index 4015f99fc..e53373029 100644
--- a/internal/service/register.go
+++ b/internal/service/register.go
@@ -4,12 +4,14 @@ import (
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promauto"
 	"gitlab.com/gitlab-org/gitaly/internal/config"
+	"gitlab.com/gitlab-org/gitaly/internal/middleware/errorhandler"
 	"gitlab.com/gitlab-org/gitaly/internal/rubyserver"
 	"gitlab.com/gitlab-org/gitaly/internal/service/blob"
 	"gitlab.com/gitlab-org/gitaly/internal/service/cleanup"
 	"gitlab.com/gitlab-org/gitaly/internal/service/commit"
 	"gitlab.com/gitlab-org/gitaly/internal/service/conflicts"
 	"gitlab.com/gitlab-org/gitaly/internal/service/diff"
+	"gitlab.com/gitlab-org/gitaly/internal/service/health"
 	hook "gitlab.com/gitlab-org/gitaly/internal/service/hooks"
 	"gitlab.com/gitlab-org/gitaly/internal/service/internalgitaly"
 	"gitlab.com/gitlab-org/gitaly/internal/service/namespace"
@@ -24,8 +26,6 @@ import (
 	"gitlab.com/gitlab-org/gitaly/internal/service/wiki"
 	"gitlab.com/gitlab-org/gitaly/proto/go/gitalypb"
 	"google.golang.org/grpc"
-	"google.golang.org/grpc/health"
-	healthpb "google.golang.org/grpc/health/grpc_health_v1"
 )
 
 var (
@@ -52,7 +52,7 @@ var (
 
 // RegisterAll will register all the known grpc services with
 // the specified grpc service instance
-func RegisterAll(grpcServer *grpc.Server, cfg config.Cfg, rubyServer *rubyserver.Server) {
+func RegisterAll(grpcServer *grpc.Server, cfg config.Cfg, errorTracker *errorhandler.Errors, rubyServer *rubyserver.Server) {
 	gitalypb.RegisterBlobServiceServer(grpcServer, blob.NewServer(rubyServer))
 	gitalypb.RegisterCleanupServiceServer(grpcServer, cleanup.NewServer())
 	gitalypb.RegisterCommitServiceServer(grpcServer, commit.NewServer())
@@ -75,5 +75,5 @@ func RegisterAll(grpcServer *grpc.Server, cfg config.Cfg, rubyServer *rubyserver
 	gitalypb.RegisterHookServiceServer(grpcServer, hook.NewServer())
 	gitalypb.RegisterInternalGitalyServer(grpcServer, internalgitaly.NewServer(config.Config.Storages))
 
-	healthpb.RegisterHealthServer(grpcServer, health.NewServer())
+	gitalypb.RegisterHealthServer(grpcServer, health.NewServer(errorTracker))
 }
diff --git a/proto/go/gitalypb/health.pb.go b/proto/go/gitalypb/health.pb.go
new file mode 100644
index 000000000..6182737d9
--- /dev/null
+++ b/proto/go/gitalypb/health.pb.go
@@ -0,0 +1,305 @@
+// Code generated by protoc-gen-go. DO NOT EDIT.
+// source: health.proto
+
+package gitalypb
+
+import (
+	context "context"
+	fmt "fmt"
+	proto "github.com/golang/protobuf/proto"
+	grpc "google.golang.org/grpc"
+	codes "google.golang.org/grpc/codes"
+	status "google.golang.org/grpc/status"
+	math "math"
+)
+
+// Reference imports to suppress errors if they are not otherwise used.
+var _ = proto.Marshal
+var _ = fmt.Errorf
+var _ = math.Inf
+
+// This is a compile-time assertion to ensure that this generated file
+// is compatible with the proto package it is being compiled against.
+// A compilation error at this line likely means your copy of the
+// proto package needs to be updated.
+const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package
+
+type HealthCheckResponse_ServingStatus int32
+
+const (
+	HealthCheckResponse_UNKNOWN     HealthCheckResponse_ServingStatus = 0
+	HealthCheckResponse_SERVING     HealthCheckResponse_ServingStatus = 1
+	HealthCheckResponse_NOT_SERVING HealthCheckResponse_ServingStatus = 2
+)
+
+var HealthCheckResponse_ServingStatus_name = map[int32]string{
+	0: "UNKNOWN",
+	1: "SERVING",
+	2: "NOT_SERVING",
+}
+
+var HealthCheckResponse_ServingStatus_value = map[string]int32{
+	"UNKNOWN":     0,
+	"SERVING":     1,
+	"NOT_SERVING": 2,
+}
+
+func (x HealthCheckResponse_ServingStatus) String() string {
+	return proto.EnumName(HealthCheckResponse_ServingStatus_name, int32(x))
+}
+
+func (HealthCheckResponse_ServingStatus) EnumDescriptor() ([]byte, []int) {
+	return fileDescriptor_fdbebe66dda7cb29, []int{1, 0}
+}
+
+type HealthCheckRequest struct {
+	Service              string   `protobuf:"bytes,1,opt,name=service,proto3" json:"service,omitempty"`
+	XXX_NoUnkeyedLiteral struct{} `json:"-"`
+	XXX_unrecognized     []byte   `json:"-"`
+	XXX_sizecache        int32    `json:"-"`
+}
+
+func (m *HealthCheckRequest) Reset()         { *m = HealthCheckRequest{} }
+func (m *HealthCheckRequest) String() string { return proto.CompactTextString(m) }
+func (*HealthCheckRequest) ProtoMessage()    {}
+func (*HealthCheckRequest) Descriptor() ([]byte, []int) {
+	return fileDescriptor_fdbebe66dda7cb29, []int{0}
+}
+
+func (m *HealthCheckRequest) XXX_Unmarshal(b []byte) error {
+	return xxx_messageInfo_HealthCheckRequest.Unmarshal(m, b)
+}
+func (m *HealthCheckRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	return xxx_messageInfo_HealthCheckRequest.Marshal(b, m, deterministic)
+}
+func (m *HealthCheckRequest) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_HealthCheckRequest.Merge(m, src)
+}
+func (m *HealthCheckRequest) XXX_Size() int {
+	return xxx_messageInfo_HealthCheckRequest.Size(m)
+}
+func (m *HealthCheckRequest) XXX_DiscardUnknown() {
+	xxx_messageInfo_HealthCheckRequest.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_HealthCheckRequest proto.InternalMessageInfo
+
+func (m *HealthCheckRequest) GetService() string {
+	if m != nil {
+		return m.Service
+	}
+	return ""
+}
+
+type HealthCheckResponse struct {
+	Status               HealthCheckResponse_ServingStatus `protobuf:"varint,1,opt,name=status,proto3,enum=gitaly.HealthCheckResponse_ServingStatus" json:"status,omitempty"`
+	XXX_NoUnkeyedLiteral struct{}                          `json:"-"`
+	XXX_unrecognized     []byte                            `json:"-"`
+	XXX_sizecache        int32                             `json:"-"`
+}
+
+func (m *HealthCheckResponse) Reset()         { *m = HealthCheckResponse{} }
+func (m *HealthCheckResponse) String() string { return proto.CompactTextString(m) }
+func (*HealthCheckResponse) ProtoMessage()    {}
+func (*HealthCheckResponse) Descriptor() ([]byte, []int) {
+	return fileDescriptor_fdbebe66dda7cb29, []int{1}
+}
+
+func (m *HealthCheckResponse) XXX_Unmarshal(b []byte) error {
+	return xxx_messageInfo_HealthCheckResponse.Unmarshal(m, b)
+}
+func (m *HealthCheckResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	return xxx_messageInfo_HealthCheckResponse.Marshal(b, m, deterministic)
+}
+func (m *HealthCheckResponse) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_HealthCheckResponse.Merge(m, src)
+}
+func (m *HealthCheckResponse) XXX_Size() int {
+	return xxx_messageInfo_HealthCheckResponse.Size(m)
+}
+func (m *HealthCheckResponse) XXX_DiscardUnknown() {
+	xxx_messageInfo_HealthCheckResponse.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_HealthCheckResponse proto.InternalMessageInfo
+
+func (m *HealthCheckResponse) GetStatus() HealthCheckResponse_ServingStatus {
+	if m != nil {
+		return m.Status
+	}
+	return HealthCheckResponse_UNKNOWN
+}
+
+func init() {
+	proto.RegisterEnum("gitaly.HealthCheckResponse_ServingStatus", HealthCheckResponse_ServingStatus_name, HealthCheckResponse_ServingStatus_value)
+	proto.RegisterType((*HealthCheckRequest)(nil), "gitaly.HealthCheckRequest")
+	proto.RegisterType((*HealthCheckResponse)(nil), "gitaly.HealthCheckResponse")
+}
+
+func init() { proto.RegisterFile("health.proto", fileDescriptor_fdbebe66dda7cb29) }
+
+var fileDescriptor_fdbebe66dda7cb29 = []byte{
+	// 275 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0xe2, 0xc9, 0x48, 0x4d, 0xcc,
+	0x29, 0xc9, 0xd0, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17, 0x62, 0x4b, 0xcf, 0x2c, 0x49, 0xcc, 0xa9,
+	0x94, 0xe2, 0xca, 0xc9, 0xcc, 0x2b, 0x81, 0x88, 0x49, 0xf1, 0x14, 0x67, 0x24, 0x16, 0xa5, 0xa6,
+	0x40, 0x78, 0x4a, 0x7a, 0x5c, 0x42, 0x1e, 0x60, 0x1d, 0xce, 0x19, 0xa9, 0xc9, 0xd9, 0x41, 0xa9,
+	0x85, 0xa5, 0xa9, 0xc5, 0x25, 0x42, 0x12, 0x5c, 0xec, 0xc5, 0xa9, 0x45, 0x65, 0x99, 0xc9, 0xa9,
+	0x12, 0x8c, 0x0a, 0x8c, 0x1a, 0x9c, 0x41, 0x30, 0xae, 0xd2, 0x14, 0x46, 0x2e, 0x61, 0x14, 0x0d,
+	0xc5, 0x05, 0xf9, 0x79, 0xc5, 0xa9, 0x42, 0x8e, 0x5c, 0x6c, 0xc5, 0x25, 0x89, 0x25, 0xa5, 0xc5,
+	0x60, 0x0d, 0x7c, 0x46, 0x9a, 0x7a, 0x10, 0xab, 0xf5, 0xb0, 0x28, 0xd6, 0x0b, 0x06, 0x19, 0x96,
+	0x97, 0x1e, 0x0c, 0xd6, 0x10, 0x04, 0xd5, 0xa8, 0x64, 0xc5, 0xc5, 0x8b, 0x22, 0x21, 0xc4, 0xcd,
+	0xc5, 0x1e, 0xea, 0xe7, 0xed, 0xe7, 0x1f, 0xee, 0x27, 0xc0, 0x00, 0xe2, 0x04, 0xbb, 0x06, 0x85,
+	0x79, 0xfa, 0xb9, 0x0b, 0x30, 0x0a, 0xf1, 0x73, 0x71, 0xfb, 0xf9, 0x87, 0xc4, 0xc3, 0x04, 0x98,
+	0x8c, 0x16, 0x31, 0x72, 0xb1, 0x41, 0x6c, 0x12, 0xf2, 0xe2, 0x62, 0x05, 0xdb, 0x26, 0x24, 0x85,
+	0xd5, 0x09, 0x60, 0x0f, 0x4a, 0x49, 0xe3, 0x71, 0x9e, 0x12, 0xc7, 0xaf, 0xe9, 0x1a, 0x2c, 0x1c,
+	0x4c, 0x02, 0x8c, 0x42, 0x3e, 0x5c, 0xac, 0xe1, 0x89, 0x25, 0xc9, 0x19, 0x54, 0x30, 0xcb, 0x80,
+	0xd1, 0xc9, 0x20, 0x0a, 0xa4, 0x32, 0x27, 0x31, 0x49, 0x2f, 0x39, 0x3f, 0x57, 0x1f, 0xc2, 0xd4,
+	0xcd, 0x2f, 0x4a, 0xd7, 0x87, 0xe8, 0xd7, 0x07, 0xc7, 0x88, 0x7e, 0x7a, 0x3e, 0x94, 0x5f, 0x90,
+	0x94, 0xc4, 0x06, 0x16, 0x32, 0x06, 0x04, 0x00, 0x00, 0xff, 0xff, 0xe0, 0xea, 0xf1, 0xbc, 0xd6,
+	0x01, 0x00, 0x00,
+}
+
+// Reference imports to suppress errors if they are not otherwise used.
+var _ context.Context
+var _ grpc.ClientConn
+
+// This is a compile-time assertion to ensure that this generated file
+// is compatible with the grpc package it is being compiled against.
+const _ = grpc.SupportPackageIsVersion4
+
+// HealthClient is the client API for Health service.
+//
+// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream.
+type HealthClient interface {
+	Check(ctx context.Context, in *HealthCheckRequest, opts ...grpc.CallOption) (*HealthCheckResponse, error)
+	Watch(ctx context.Context, in *HealthCheckRequest, opts ...grpc.CallOption) (Health_WatchClient, error)
+}
+
+type healthClient struct {
+	cc *grpc.ClientConn
+}
+
+func NewHealthClient(cc *grpc.ClientConn) HealthClient {
+	return &healthClient{cc}
+}
+
+func (c *healthClient) Check(ctx context.Context, in *HealthCheckRequest, opts ...grpc.CallOption) (*HealthCheckResponse, error) {
+	out := new(HealthCheckResponse)
+	err := c.cc.Invoke(ctx, "/gitaly.Health/Check", in, out, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (c *healthClient) Watch(ctx context.Context, in *HealthCheckRequest, opts ...grpc.CallOption) (Health_WatchClient, error) {
+	stream, err := c.cc.NewStream(ctx, &_Health_serviceDesc.Streams[0], "/gitaly.Health/Watch", opts...)
+	if err != nil {
+		return nil, err
+	}
+	x := &healthWatchClient{stream}
+	if err := x.ClientStream.SendMsg(in); err != nil {
+		return nil, err
+	}
+	if err := x.ClientStream.CloseSend(); err != nil {
+		return nil, err
+	}
+	return x, nil
+}
+
+type Health_WatchClient interface {
+	Recv() (*HealthCheckResponse, error)
+	grpc.ClientStream
+}
+
+type healthWatchClient struct {
+	grpc.ClientStream
+}
+
+func (x *healthWatchClient) Recv() (*HealthCheckResponse, error) {
+	m := new(HealthCheckResponse)
+	if err := x.ClientStream.RecvMsg(m); err != nil {
+		return nil, err
+	}
+	return m, nil
+}
+
+// HealthServer is the server API for Health service.
+type HealthServer interface {
+	Check(context.Context, *HealthCheckRequest) (*HealthCheckResponse, error)
+	Watch(*HealthCheckRequest, Health_WatchServer) error
+}
+
+// UnimplementedHealthServer can be embedded to have forward compatible implementations.
+type UnimplementedHealthServer struct {
+}
+
+func (*UnimplementedHealthServer) Check(ctx context.Context, req *HealthCheckRequest) (*HealthCheckResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method Check not implemented")
+}
+func (*UnimplementedHealthServer) Watch(req *HealthCheckRequest, srv Health_WatchServer) error {
+	return status.Errorf(codes.Unimplemented, "method Watch not implemented")
+}
+
+func RegisterHealthServer(s *grpc.Server, srv HealthServer) {
+	s.RegisterService(&_Health_serviceDesc, srv)
+}
+
+func _Health_Check_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(HealthCheckRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(HealthServer).Check(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: "/gitaly.Health/Check",
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(HealthServer).Check(ctx, req.(*HealthCheckRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+func _Health_Watch_Handler(srv interface{}, stream grpc.ServerStream) error {
+	m := new(HealthCheckRequest)
+	if err := stream.RecvMsg(m); err != nil {
+		return err
+	}
+	return srv.(HealthServer).Watch(m, &healthWatchServer{stream})
+}
+
+type Health_WatchServer interface {
+	Send(*HealthCheckResponse) error
+	grpc.ServerStream
+}
+
+type healthWatchServer struct {
+	grpc.ServerStream
+}
+
+func (x *healthWatchServer) Send(m *HealthCheckResponse) error {
+	return x.ServerStream.SendMsg(m)
+}
+
+var _Health_serviceDesc = grpc.ServiceDesc{
+	ServiceName: "gitaly.Health",
+	HandlerType: (*HealthServer)(nil),
+	Methods: []grpc.MethodDesc{
+		{
+			MethodName: "Check",
+			Handler:    _Health_Check_Handler,
+		},
+	},
+	Streams: []grpc.StreamDesc{
+		{
+			StreamName:    "Watch",
+			Handler:       _Health_Watch_Handler,
+			ServerStreams: true,
+		},
+	},
+	Metadata: "health.proto",
+}
diff --git a/proto/go/gitalypb/protolist.go b/proto/go/gitalypb/protolist.go
index a15916f70..a46b6e48f 100644
--- a/proto/go/gitalypb/protolist.go
+++ b/proto/go/gitalypb/protolist.go
@@ -9,6 +9,7 @@ var GitalyProtos = []string{
 	"commit.proto",
 	"conflicts.proto",
 	"diff.proto",
+	"health.proto",
 	"hook.proto",
 	"internal.proto",
 	"lint.proto",
diff --git a/proto/health.proto b/proto/health.proto
new file mode 100644
index 000000000..16a063ee8
--- /dev/null
+++ b/proto/health.proto
@@ -0,0 +1,37 @@
+syntax = "proto3";
+
+package gitaly;
+
+option go_package = "gitlab.com/gitlab-org/gitaly/proto/go/gitalypb";
+
+import "lint.proto";
+import "shared.proto";
+
+service Health {
+  rpc Check(HealthCheckRequest) returns (HealthCheckResponse){
+    option (op_type) = {
+      op: ACCESSOR
+      scope_level: SERVER
+    };
+  };
+
+  rpc Watch(HealthCheckRequest) returns (stream HealthCheckResponse){
+    option (op_type) = {
+      op: ACCESSOR
+      scope_level: SERVER
+    };
+  };
+}
+
+message HealthCheckRequest {
+  string service = 1;
+}
+
+message HealthCheckResponse {
+  enum ServingStatus {
+    UNKNOWN = 0;
+    SERVING = 1;
+    NOT_SERVING = 2;
+  }
+  ServingStatus status = 1;
+}
diff --git a/ruby/proto/gitaly.rb b/ruby/proto/gitaly.rb
index 9c80cea63..022b159e4 100644
--- a/ruby/proto/gitaly.rb
+++ b/ruby/proto/gitaly.rb
@@ -13,6 +13,8 @@ require 'gitaly/conflicts_services_pb'
 
 require 'gitaly/diff_services_pb'
 
+require 'gitaly/health_services_pb'
+
 require 'gitaly/hook_services_pb'
 
 require 'gitaly/internal_services_pb'
diff --git a/ruby/proto/gitaly/health_pb.rb b/ruby/proto/gitaly/health_pb.rb
new file mode 100644
index 000000000..ea95aef86
--- /dev/null
+++ b/ruby/proto/gitaly/health_pb.rb
@@ -0,0 +1,26 @@
+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+# source: health.proto
+
+require 'google/protobuf'
+
+require 'lint_pb'
+require 'shared_pb'
+Google::Protobuf::DescriptorPool.generated_pool.build do
+  add_message "gitaly.HealthCheckRequest" do
+    optional :service, :string, 1
+  end
+  add_message "gitaly.HealthCheckResponse" do
+    optional :status, :enum, 1, "gitaly.HealthCheckResponse.ServingStatus"
+  end
+  add_enum "gitaly.HealthCheckResponse.ServingStatus" do
+    value :UNKNOWN, 0
+    value :SERVING, 1
+    value :NOT_SERVING, 2
+  end
+end
+
+module Gitaly
+  HealthCheckRequest = Google::Protobuf::DescriptorPool.generated_pool.lookup("gitaly.HealthCheckRequest").msgclass
+  HealthCheckResponse = Google::Protobuf::DescriptorPool.generated_pool.lookup("gitaly.HealthCheckResponse").msgclass
+  HealthCheckResponse::ServingStatus = Google::Protobuf::DescriptorPool.generated_pool.lookup("gitaly.HealthCheckResponse.ServingStatus").enummodule
+end
diff --git a/ruby/proto/gitaly/health_services_pb.rb b/ruby/proto/gitaly/health_services_pb.rb
new file mode 100644
index 000000000..11f24e5e9
--- /dev/null
+++ b/ruby/proto/gitaly/health_services_pb.rb
@@ -0,0 +1,23 @@
+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+# Source: health.proto for package 'gitaly'
+
+require 'grpc'
+require 'health_pb'
+
+module Gitaly
+  module Health
+    class Service
+
+      include GRPC::GenericService
+
+      self.marshal_class_method = :encode
+      self.unmarshal_class_method = :decode
+      self.service_name = 'gitaly.Health'
+
+      rpc :Check, HealthCheckRequest, HealthCheckResponse
+      rpc :Watch, HealthCheckRequest, stream(HealthCheckResponse)
+    end
+
+    Stub = Service.rpc_stub_class
+  end
+end