Add client feature logging and metrics

3 files changed, 102 insertions, 0 deletions

diff --git a/CHANGELOG.md b/CHANGELOG.md
index fa4f3fc85..4198f2564 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -8,6 +8,8 @@ v0.44.0
   https://gitlab.com/gitlab-org/gitaly/merge_requests/385
 - Implement Raw{Diff,Patch} RPCs
   https://gitlab.com/gitlab-org/gitaly/merge_requests/381
+- Add client feature logging and metrics
+  https://gitlab.com/gitlab-org/gitaly/merge_requests/392
 
 v0.43.0
 
diff --git a/internal/middleware/metadatahandler/metadatahandler.go b/internal/middleware/metadatahandler/metadatahandler.go
new file mode 100644
index 000000000..3f70f24bd
--- /dev/null
+++ b/internal/middleware/metadatahandler/metadatahandler.go
@@ -0,0 +1,97 @@
+package metadatahandler
+
+import (
+	"github.com/grpc-ecosystem/go-grpc-middleware/tags"
+	"github.com/prometheus/client_golang/prometheus"
+	"golang.org/x/net/context"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/metadata"
+)
+
+var (
+	requests = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: "gitaly",
+			Subsystem: "service",
+			Name:      "client_requests",
+			Help:      "Counter of client requests received by client, call_site and response code",
+		},
+		[]string{"client_name", "call_site", "grpc_code"},
+	)
+)
+
+func init() {
+	prometheus.MustRegister(requests)
+}
+
+// CallSiteKey is the key used in ctx_tags to store the client feature
+const CallSiteKey = "grpc.meta.call_site"
+
+// ClientNameKey is the key used in ctx_tags to store the client name
+const ClientNameKey = "grpc.meta.client_name"
+
+// Unknown client and feature. Matches the prometheus grpc unknown value
+const unknownValue = "unknown"
+
+func getFromMD(md metadata.MD, header string) string {
+	values := md[header]
+	if len(values) != 1 {
+		return ""
+	}
+
+	return values[0]
+}
+
+// addMetadataTags extracts metadata from the connection headers and add it to the
+// ctx_tags, if it is set. Returns values appropriate for use with prometheus labels,
+// using `unknown` if a value is not set
+func addMetadataTags(ctx context.Context) (clientName string, callSite string) {
+	clientName = unknownValue
+	callSite = unknownValue
+
+	md, ok := metadata.FromIncomingContext(ctx)
+	if !ok {
+		return clientName, callSite
+	}
+
+	tags := grpc_ctxtags.Extract(ctx)
+
+	metadata := getFromMD(md, "call_site")
+	if metadata != "" {
+		callSite = metadata
+		tags.Set(CallSiteKey, metadata)
+	}
+
+	metadata = getFromMD(md, "client_name")
+	if metadata != "" {
+		clientName = metadata
+		tags.Set(ClientNameKey, metadata)
+	}
+
+	return clientName, callSite
+}
+
+// UnaryInterceptor returns a Unary Interceptor
+func UnaryInterceptor(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) {
+	clientName, callSite := addMetadataTags(ctx)
+
+	res, err := handler(ctx, req)
+
+	grpcCode := grpc.Code(err)
+	requests.WithLabelValues(clientName, callSite, grpcCode.String()).Inc()
+
+	return res, err
+}
+
+// StreamInterceptor returns a Stream Interceptor
+func StreamInterceptor(srv interface{}, stream grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) error {
+	ctx := stream.Context()
+	clientName, callSite := addMetadataTags(ctx)
+
+	err := handler(srv, stream)
+
+	grpcCode := grpc.Code(err)
+	requests.WithLabelValues(clientName, callSite, grpcCode.String()).Inc()
+
+	return err
+}
diff --git a/internal/server/server.go b/internal/server/server.go
index 33d8da3fd..bf855cc3a 100644
--- a/internal/server/server.go
+++ b/internal/server/server.go
@@ -6,6 +6,7 @@ import (
 	"gitlab.com/gitlab-org/gitaly/internal/helper/fieldextractors"
 	"gitlab.com/gitlab-org/gitaly/internal/middleware/cancelhandler"
 	"gitlab.com/gitlab-org/gitaly/internal/middleware/limithandler"
+	"gitlab.com/gitlab-org/gitaly/internal/middleware/metadatahandler"
 	"gitlab.com/gitlab-org/gitaly/internal/middleware/panichandler"
 	"gitlab.com/gitlab-org/gitaly/internal/middleware/sentryhandler"
 	"gitlab.com/gitlab-org/gitaly/internal/rubyserver"
@@ -34,6 +35,7 @@ func New(rubyServer *rubyserver.Server) *grpc.Server {
 	server := grpc.NewServer(
 		grpc.StreamInterceptor(grpc_middleware.ChainStreamServer(
 			grpc_ctxtags.StreamServerInterceptor(ctxTagOpts...),
+			metadatahandler.StreamInterceptor,
 			grpc_prometheus.StreamServerInterceptor,
 			grpc_logrus.StreamServerInterceptor(logrusEntry),
 			sentryhandler.StreamLogHandler,
@@ -46,6 +48,7 @@ func New(rubyServer *rubyserver.Server) *grpc.Server {
 		)),
 		grpc.UnaryInterceptor(grpc_middleware.ChainUnaryServer(
 			grpc_ctxtags.UnaryServerInterceptor(ctxTagOpts...),
+			metadatahandler.UnaryInterceptor,
 			grpc_prometheus.UnaryServerInterceptor,
 			grpc_logrus.UnaryServerInterceptor(logrusEntry),
 			sentryhandler.UnaryLogHandler,