Add a gauge to track in flight commands

Running commands is not something that's currently tracked. Adding a
Gauge in prometheus to track it and allow this to be trackend and
potentially alerted on.

Closes https://gitlab.com/gitlab-org/gitaly/issues/1844

2 files changed, 16 insertions, 0 deletions

diff --git a/internal/command/command.go b/internal/command/command.go
index a89be98cf..ae374f3a8 100644
--- a/internal/command/command.go
+++ b/internal/command/command.go
@@ -243,6 +243,7 @@ func New(ctx context.Context, cmd *exec.Cmd, stdin io.Reader, stdout, stderr io.
 	if err := cmd.Start(); err != nil {
 		return nil, fmt.Errorf("GitCommand: start %v: %v", cmd.Args, err)
 	}
+	inFlightCommandGauge.Inc()
 
 	// The goroutine below is responsible for terminating and reaping the
 	// process when ctx is canceled.
@@ -359,6 +360,7 @@ func (c *Command) wait() {
 		}
 	}
 
+	inFlightCommandGauge.Dec()
 	c.logProcessComplete(c.context, exitCode)
 
 	if w := c.stderrCloser; w != nil {
diff --git a/internal/command/metrics.go b/internal/command/metrics.go
new file mode 100644
index 000000000..a089b8cd8
--- /dev/null
+++ b/internal/command/metrics.go
@@ -0,0 +1,14 @@
+package command
+
+import "github.com/prometheus/client_golang/prometheus"
+
+var inFlightCommandGauge = prometheus.NewGauge(
+	prometheus.GaugeOpts{
+		Name: "gitaly_commands_running",
+		Help: "Total number of processes currently being executed",
+	},
+)
+
+func init() {
+	prometheus.MustRegister(inFlightCommandGauge)
+}