Add metric for measuring PumaWorkerKiller activity

PumaWorkerKiller is used for periodically checking and killing
workers (the biggest one) if overall memory reaches specified
limit. This metric allows us to watch number of killed workers.

2 files changed, 27 insertions, 0 deletions

diff --git a/lib/gitlab/cluster/puma_worker_killer_initializer.rb b/lib/gitlab/cluster/puma_worker_killer_initializer.rb
index 4ed9a9a02ab..4affc52b7b0 100644
--- a/lib/gitlab/cluster/puma_worker_killer_initializer.rb
+++ b/lib/gitlab/cluster/puma_worker_killer_initializer.rb
@@ -27,6 +27,9 @@ module Gitlab
           # is restarted already, thus periodically restarting workers shouldn't be
           # needed.
           config.rolling_restart_frequency = false
+
+          observer = Gitlab::Cluster::PumaWorkerKillerObserver.new
+          config.pre_term = observer.callback
         end
 
         PumaWorkerKiller.start
diff --git a/lib/gitlab/cluster/puma_worker_killer_observer.rb b/lib/gitlab/cluster/puma_worker_killer_observer.rb
new file mode 100644
index 00000000000..3b4ebc3fbae
--- /dev/null
+++ b/lib/gitlab/cluster/puma_worker_killer_observer.rb
@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Cluster
+    class PumaWorkerKillerObserver
+      def initialize
+        @counter = Gitlab::Metrics.counter(:puma_killer_terminations_total, 'Number of workers terminated by PumaWorkerKiller')
+      end
+
+      # returns the Proc to be used as the observer callback block
+      def callback
+        method(:log_termination)
+      end
+
+      private
+
+      def log_termination(worker)
+        labels = { worker: "worker_#{worker.index}" }
+
+        @counter.increment(labels)
+      end
+    end
+  end
+end