app/services/clusters/applications/prometheus_health_check_service.rb


1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85

# frozen_string_literal: true

module Clusters
  module Applications
    class PrometheusHealthCheckService
      include Gitlab::Utils::StrongMemoize
      include Gitlab::Routing

      def initialize(cluster)
        @cluster = cluster
        @logger = Gitlab::AppJsonLogger.build
      end

      def execute
        raise 'Invalid cluster type. Only project types are allowed.' unless @cluster.project_type?

        return unless prometheus_application.installed?

        project = @cluster.clusterable

        @logger.info(
          message: 'Prometheus health check',
          cluster_id: @cluster.id,
          newly_unhealthy: became_unhealthy?,
          currently_healthy: currently_healthy?,
          was_healthy: was_healthy?
        )

        send_notification(project) if became_unhealthy?

        prometheus_application.update_columns(healthy: currently_healthy?) if health_changed?
      end

      private

      def prometheus_application
        strong_memoize(:prometheus_application) do
          @cluster.application_prometheus
        end
      end

      def currently_healthy?
        strong_memoize(:currently_healthy) do
          prometheus_application.prometheus_client.healthy?
        end
      end

      def became_unhealthy?
        strong_memoize(:became_unhealthy) do
          (was_healthy? || was_healthy?.nil?) && !currently_healthy?
        end
      end

      def was_healthy?
        strong_memoize(:was_healthy) do
          prometheus_application.healthy
        end
      end

      def health_changed?
        was_healthy? != currently_healthy?
      end

      def send_notification(project)
        notification_payload = build_notification_payload(project)
        token = project.alerts_service.data.token
        Projects::Alerting::NotifyService.new(project, nil, notification_payload).execute(token)
        @logger.info(message: 'Successfully notified of Prometheus newly unhealthy', cluster_id: @cluster.id, project_id: project.id)
      end

      def build_notification_payload(project)
        cluster_path = namespace_project_cluster_path(
          project_id: project.path,
          namespace_id: project.namespace.path,
          id: @cluster.id
        )

        {
          title: "Prometheus is Unhealthy. Cluster Name: #{@cluster.name}",
          description: "Prometheus is unhealthy for the cluster: [#{@cluster.name}](#{cluster_path}) attached to project #{project.name}."
        }
      end
    end
  end
end