lib/gitlab/memory/watchdog/event_reporter.rb


1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68

# frozen_string_literal: true

module Gitlab
  module Memory
    class Watchdog
      class EventReporter
        include ::Gitlab::Utils::StrongMemoize

        attr_reader :logger

        def initialize(logger: Gitlab::AppLogger)
          @logger = logger
          init_prometheus_metrics
        end

        def started(labels = {})
          logger.info(message: 'started', **log_labels(labels))
        end

        def stopped(labels = {})
          logger.info(message: 'stopped', **log_labels(labels))
        end

        def threshold_violated(monitor_name)
          @counter_violations.increment(reason: monitor_name)
        end

        def strikes_exceeded(monitor_name, labels = {})
          logger.warn(log_labels(labels))

          @counter_violations_handled.increment(reason: monitor_name)
        end

        private

        def log_labels(extra = {})
          extra.merge(
            pid: $$,
            worker_id: worker_id,
            memwd_rss_bytes: process_rss_bytes
          )
        end

        def process_rss_bytes
          Gitlab::Metrics::System.memory_usage_rss[:total]
        end

        def worker_id
          ::Prometheus::PidProvider.worker_id
        end

        def init_prometheus_metrics
          default_labels = { pid: worker_id }
          @counter_violations = Gitlab::Metrics.counter(
            :gitlab_memwd_violations_total,
            'Total number of times a Ruby process violated a memory threshold',
            default_labels
          )
          @counter_violations_handled = Gitlab::Metrics.counter(
            :gitlab_memwd_violations_handled_total,
            'Total number of times Ruby process memory violations were handled',
            default_labels
          )
        end
      end
    end
  end
end