Add latest changes from gitlab-org/gitlab@master

4 files changed, 88 insertions, 45 deletions

diff --git a/lib/gitlab/memory/watchdog.rb b/lib/gitlab/memory/watchdog.rb
index 435b416e7e9..2e0add73478 100644
--- a/lib/gitlab/memory/watchdog.rb
+++ b/lib/gitlab/memory/watchdog.rb
@@ -50,8 +50,6 @@ module Gitlab
       def initialize
         @configuration = Configuration.new
         @alive = true
-
-        init_prometheus_metrics
       end
 
       ##
@@ -62,7 +60,7 @@ module Gitlab
       end
 
       def call
-        logger.info(log_labels.merge(message: 'started'))
+        event_reporter.started(log_labels)
 
         while @alive
           sleep(sleep_time_seconds)
@@ -70,7 +68,7 @@ module Gitlab
           monitor if Feature.enabled?(:gitlab_memory_watchdog, type: :ops)
         end
 
-        logger.info(log_labels.merge(message: 'stopped'))
+        event_reporter.stopped(log_labels)
       end
 
       def stop
@@ -85,18 +83,16 @@ module Gitlab
 
           next unless result.threshold_violated?
 
-          @counter_violations.increment(reason: result.monitor_name)
+          event_reporter.threshold_violated(result.monitor_name)
 
           next unless result.strikes_exceeded?
 
-          @alive = !memory_limit_exceeded_callback(result.monitor_name, result.payload)
+          @alive = !strike_exceeded_callback(result.monitor_name, result.payload)
         end
       end
 
-      def memory_limit_exceeded_callback(monitor_name, monitor_payload)
-        all_labels = log_labels.merge(monitor_payload)
-        logger.warn(all_labels)
-        @counter_violations_handled.increment(reason: monitor_name)
+      def strike_exceeded_callback(monitor_name, monitor_payload)
+        event_reporter.strikes_exceeded(monitor_name, log_labels(monitor_payload))
 
         Gitlab::Memory::Reports::HeapDump.enqueue! if @configuration.write_heap_dumps?
 
@@ -111,43 +107,18 @@ module Gitlab
         @configuration.handler
       end
 
-      def logger
-        @configuration.logger
+      def event_reporter
+        @configuration.event_reporter
       end
 
       def sleep_time_seconds
         @configuration.sleep_time_seconds
       end
 
-      def log_labels
-        {
-          pid: $$,
-          worker_id: worker_id,
+      def log_labels(extra = {})
+        extra.merge(
           memwd_handler_class: handler.class.name,
-          memwd_sleep_time_s: sleep_time_seconds,
-          memwd_rss_bytes: process_rss_bytes
-        }
-      end
-
-      def process_rss_bytes
-        Gitlab::Metrics::System.memory_usage_rss[:total]
-      end
-
-      def worker_id
-        ::Prometheus::PidProvider.worker_id
-      end
-
-      def init_prometheus_metrics
-        default_labels = { pid: worker_id }
-        @counter_violations = Gitlab::Metrics.counter(
-          :gitlab_memwd_violations_total,
-          'Total number of times a Ruby process violated a memory threshold',
-          default_labels
-        )
-        @counter_violations_handled = Gitlab::Metrics.counter(
-          :gitlab_memwd_violations_handled_total,
-          'Total number of times Ruby process memory violations were handled',
-          default_labels
+          memwd_sleep_time_s: sleep_time_seconds
         )
       end
     end
diff --git a/lib/gitlab/memory/watchdog/configuration.rb b/lib/gitlab/memory/watchdog/configuration.rb
index 885772d6119..980b6bf750b 100644
--- a/lib/gitlab/memory/watchdog/configuration.rb
+++ b/lib/gitlab/memory/watchdog/configuration.rb
@@ -35,7 +35,7 @@ module Gitlab
 
         DEFAULT_SLEEP_TIME_SECONDS = 60
 
-        attr_writer :logger, :handler, :sleep_time_seconds, :write_heap_dumps
+        attr_writer :event_reporter, :handler, :sleep_time_seconds, :write_heap_dumps
 
         def monitors
           @monitor_stack ||= MonitorStack.new
@@ -47,8 +47,8 @@ module Gitlab
           @handler ||= NullHandler.instance
         end
 
-        def logger
-          @logger ||= Gitlab::Logger.new($stdout)
+        def event_reporter
+          @event_reporter ||= EventReporter.new
         end
 
         # Used to control the frequency with which the watchdog will wake up and poll the GC.
diff --git a/lib/gitlab/memory/watchdog/configurator.rb b/lib/gitlab/memory/watchdog/configurator.rb
index 610d8ca9e97..880f9800d96 100644
--- a/lib/gitlab/memory/watchdog/configurator.rb
+++ b/lib/gitlab/memory/watchdog/configurator.rb
@@ -17,7 +17,6 @@ module Gitlab
         class << self
           def configure_for_puma
             ->(config) do
-              config.logger = Gitlab::AppLogger
               config.handler = Gitlab::Memory::Watchdog::PumaHandler.new
               config.write_heap_dumps = write_heap_dumps?
               config.sleep_time_seconds = ENV.fetch('GITLAB_MEMWD_SLEEP_TIME_SEC', DEFAULT_SLEEP_INTERVAL_S).to_i
@@ -27,11 +26,11 @@ module Gitlab
 
           def configure_for_sidekiq
             ->(config) do
-              config.logger = Sidekiq.logger
               config.handler = Gitlab::Memory::Watchdog::TermProcessHandler.new
               config.write_heap_dumps = write_heap_dumps?
               config.sleep_time_seconds = sidekiq_sleep_time
               config.monitors(&configure_monitors_for_sidekiq)
+              config.event_reporter = EventReporter.new(logger: ::Sidekiq.logger)
             end
           end
 
diff --git a/lib/gitlab/memory/watchdog/event_reporter.rb b/lib/gitlab/memory/watchdog/event_reporter.rb
new file mode 100644
index 00000000000..4d37a5e14fd
--- /dev/null
+++ b/lib/gitlab/memory/watchdog/event_reporter.rb
@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Memory
+    class Watchdog
+      class EventReporter
+        include ::Gitlab::Utils::StrongMemoize
+
+        attr_reader :logger
+
+        def initialize(logger: Gitlab::AppLogger)
+          @logger = logger
+        end
+
+        def started(labels = {})
+          logger.info(message: 'started', **log_labels(labels))
+        end
+
+        def stopped(labels = {})
+          logger.info(message: 'stopped', **log_labels(labels))
+        end
+
+        def threshold_violated(monitor_name)
+          counter_violations.increment(reason: monitor_name)
+        end
+
+        def strikes_exceeded(monitor_name, labels = {})
+          logger.warn(log_labels(labels))
+
+          counter_violations_handled.increment(reason: monitor_name)
+        end
+
+        private
+
+        def log_labels(extra = {})
+          extra.merge(
+            pid: $$,
+            worker_id: worker_id,
+            memwd_rss_bytes: process_rss_bytes
+          )
+        end
+
+        def process_rss_bytes
+          Gitlab::Metrics::System.memory_usage_rss[:total]
+        end
+
+        def worker_id
+          ::Prometheus::PidProvider.worker_id
+        end
+
+        def counter_violations
+          strong_memoize("counter_violations") do
+            ::Gitlab::Metrics.counter(
+              :gitlab_memwd_violations_total,
+              'Total number of times a Ruby process violated a memory threshold',
+              { pid: worker_id }
+            )
+          end
+        end
+
+        def counter_violations_handled
+          strong_memoize("counter_violations_handled") do
+            ::Gitlab::Metrics.counter(
+              :gitlab_memwd_violations_handled_total,
+              'Total number of times Ruby process memory violations were handled',
+              { pid: worker_id }
+            )
+          end
+        end
+      end
+    end
+  end
+end