Merge branch 'allow-grpc-latencies-metrics' into 'master'

Allow Gitaly to be configured to use histograms for GRPC method calls

See merge request !172

5 files changed, 39 insertions, 7 deletions

diff --git a/CHANGELOG.md b/CHANGELOG.md
index e5acd07f7..2242db64c 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -22,7 +22,9 @@ UNRELEASED
   https://gitlab.com/gitlab-org/gitaly/merge_requests/173
 - Ensure that grpc server log messages are sent to logrus
   https://gitlab.com/gitlab-org/gitaly/merge_requests/174
-  
+- Add support for GRPC Latency Histograms in Prometheus
+  https://gitlab.com/gitlab-org/gitaly/merge_requests/172
+
 v0.10.0
 
 - CommitDiff: Parse a typechange diff correctly
diff --git a/cmd/gitaly/main.go b/cmd/gitaly/main.go
index b09a7dc8d..70e880b87 100644
--- a/cmd/gitaly/main.go
+++ b/cmd/gitaly/main.go
@@ -86,6 +86,7 @@ func main() {
 
 	config.ConfigureLogging()
 	grpclog.SetLogger(log.StandardLogger())
+	config.ConfigurePrometheus()
 
 	var listeners []net.Listener
 
@@ -124,7 +125,6 @@ func main() {
 	service.RegisterAll(server)
 	reflection.Register(server)
 
-	// After all your registrations, make sure all of the Prometheus metrics are initialized.
 	grpc_prometheus.Register(server)
 
 	serverError := make(chan error, len(listeners))
diff --git a/config.toml.example b/config.toml.example
index 3778962d0..5ccbae9af 100644
--- a/config.toml.example
+++ b/config.toml.example
@@ -24,3 +24,7 @@ path = "/home/git/repositories"
 # # You can optionally configure Gitaly to output JSON-formatted log messages to stdout
 # [logging]
 # format = "json"
+
+# # You can optionally configure Gitaly to record histogram latencies on GRPC method calls
+# [prometheus]
+# grpc_latency_buckets = [0.001, 0.005, 0.025, 0.1, 0.5, 1.0, 10.0, 30.0, 60.0, 300.0, 1500.0]
diff --git a/internal/config/config.go b/internal/config/config.go
index e62f01c23..37892a2e9 100644
--- a/internal/config/config.go
+++ b/internal/config/config.go
@@ -16,11 +16,12 @@ var (
 )
 
 type config struct {
-	SocketPath           string    `toml:"socket_path" split_words:"true"`
-	ListenAddr           string    `toml:"listen_addr" split_words:"true"`
-	PrometheusListenAddr string    `toml:"prometheus_listen_addr" split_words:"true"`
-	Storages             []Storage `toml:"storage" envconfig:"storage"`
-	Logging              Logging   `toml:"logging" envconfig:"logging"`
+	SocketPath           string     `toml:"socket_path" split_words:"true"`
+	ListenAddr           string     `toml:"listen_addr" split_words:"true"`
+	PrometheusListenAddr string     `toml:"prometheus_listen_addr" split_words:"true"`
+	Storages             []Storage  `toml:"storage" envconfig:"storage"`
+	Logging              Logging    `toml:"logging" envconfig:"logging"`
+	Prometheus           Prometheus `toml:"prometheus"`
 }
 
 // Storage contains a single storage-shard
@@ -34,6 +35,11 @@ type Logging struct {
 	Format string
 }
 
+// Prometheus contains additional configuration data for prometheus
+type Prometheus struct {
+	GRPCLatencyBuckets []float64 `toml:"grpc_latency_buckets"`
+}
+
 // Load initializes the Config variable from file and the environment.
 //  Environment variables take precedence over the file.
 func Load(file io.Reader) error {
diff --git a/internal/config/prometheus.go b/internal/config/prometheus.go
new file mode 100644
index 000000000..eb5dacc5b
--- /dev/null
+++ b/internal/config/prometheus.go
@@ -0,0 +1,20 @@
+package config
+
+import (
+	"github.com/grpc-ecosystem/go-grpc-prometheus"
+	"github.com/prometheus/client_golang/prometheus"
+	log "github.com/sirupsen/logrus"
+)
+
+// ConfigurePrometheus uses the global configuration to configure prometheus
+func ConfigurePrometheus() {
+	if len(Config.Prometheus.GRPCLatencyBuckets) == 0 {
+		return
+	}
+
+	log.WithField("latencies", Config.Prometheus.GRPCLatencyBuckets).Debug("grpc prometheus histograms enabled")
+
+	grpc_prometheus.EnableHandlingTimeHistogram(func(histogramOpts *prometheus.HistogramOpts) {
+		histogramOpts.Buckets = Config.Prometheus.GRPCLatencyBuckets
+	})
+}