diff options
Diffstat (limited to 'lib/gitlab/metrics')
-rw-r--r-- | lib/gitlab/metrics/dashboard/finder.rb | 17 | ||||
-rw-r--r-- | lib/gitlab/metrics/exporter/base_exporter.rb | 3 | ||||
-rw-r--r-- | lib/gitlab/metrics/exporter/sidekiq_exporter.rb | 3 | ||||
-rw-r--r-- | lib/gitlab/metrics/exporter/web_exporter.rb | 3 | ||||
-rw-r--r-- | lib/gitlab/metrics/prometheus.rb | 2 | ||||
-rw-r--r-- | lib/gitlab/metrics/subscribers/action_cable.rb | 22 | ||||
-rw-r--r-- | lib/gitlab/metrics/subscribers/active_record.rb | 30 | ||||
-rw-r--r-- | lib/gitlab/metrics/subscribers/load_balancing.rb | 70 |
8 files changed, 125 insertions, 25 deletions
diff --git a/lib/gitlab/metrics/dashboard/finder.rb b/lib/gitlab/metrics/dashboard/finder.rb index 2c4793eb75f..c8591a81a05 100644 --- a/lib/gitlab/metrics/dashboard/finder.rb +++ b/lib/gitlab/metrics/dashboard/finder.rb @@ -7,14 +7,9 @@ module Gitlab module Metrics module Dashboard class Finder - # Dashboards that should not be part of the list of all dashboards - # displayed on the metrics dashboard page. - PREDEFINED_DASHBOARD_EXCLUSION_LIST = [ - # This dashboard is only useful in the self monitoring project. - ::Metrics::Dashboard::SelfMonitoringDashboardService, - - # This dashboard is displayed on the K8s cluster settings health page. - ::Metrics::Dashboard::ClusterDashboardService + PREDEFINED_DASHBOARD_LIST = [ + ::Metrics::Dashboard::PodDashboardService, + ::Metrics::Dashboard::SystemDashboardService ].freeze class << self @@ -90,11 +85,7 @@ module Gitlab return [self_monitoring_service] end - predefined_dashboard_services - end - - def predefined_dashboard_services - ::Metrics::Dashboard::PredefinedDashboardService.descendants - PREDEFINED_DASHBOARD_EXCLUSION_LIST + PREDEFINED_DASHBOARD_LIST end def system_service diff --git a/lib/gitlab/metrics/exporter/base_exporter.rb b/lib/gitlab/metrics/exporter/base_exporter.rb index 7111835c85a..ff8b8bf2237 100644 --- a/lib/gitlab/metrics/exporter/base_exporter.rb +++ b/lib/gitlab/metrics/exporter/base_exporter.rb @@ -1,5 +1,8 @@ # frozen_string_literal: true +require 'webrick' +require 'prometheus/client/rack/exporter' + module Gitlab module Metrics module Exporter diff --git a/lib/gitlab/metrics/exporter/sidekiq_exporter.rb b/lib/gitlab/metrics/exporter/sidekiq_exporter.rb index 36262b09b2d..4d38d9e67bf 100644 --- a/lib/gitlab/metrics/exporter/sidekiq_exporter.rb +++ b/lib/gitlab/metrics/exporter/sidekiq_exporter.rb @@ -1,8 +1,5 @@ # frozen_string_literal: true -require 'webrick' -require 'prometheus/client/rack/exporter' - module Gitlab module Metrics module Exporter diff --git a/lib/gitlab/metrics/exporter/web_exporter.rb b/lib/gitlab/metrics/exporter/web_exporter.rb index 756e6b0641a..f378577f08e 100644 --- a/lib/gitlab/metrics/exporter/web_exporter.rb +++ b/lib/gitlab/metrics/exporter/web_exporter.rb @@ -1,8 +1,5 @@ # frozen_string_literal: true -require 'webrick' -require 'prometheus/client/rack/exporter' - module Gitlab module Metrics module Exporter diff --git a/lib/gitlab/metrics/prometheus.rb b/lib/gitlab/metrics/prometheus.rb index 757762499a9..848b73792cb 100644 --- a/lib/gitlab/metrics/prometheus.rb +++ b/lib/gitlab/metrics/prometheus.rb @@ -1,7 +1,5 @@ # frozen_string_literal: true -require 'prometheus/client' - module Gitlab module Metrics module Prometheus diff --git a/lib/gitlab/metrics/subscribers/action_cable.rb b/lib/gitlab/metrics/subscribers/action_cable.rb index a9355eeae40..631b9209f14 100644 --- a/lib/gitlab/metrics/subscribers/action_cable.rb +++ b/lib/gitlab/metrics/subscribers/action_cable.rb @@ -12,6 +12,7 @@ module Gitlab TRANSMIT_SUBSCRIPTION_CONFIRMATION = :action_cable_subscription_confirmations_total TRANSMIT_SUBSCRIPTION_REJECTION = :action_cable_subscription_rejections_total BROADCAST = :action_cable_broadcasts_total + DATA_TRANSMITTED_BYTES = :action_cable_transmitted_bytes def transmit_subscription_confirmation(event) confirm_subscription_counter.increment @@ -23,6 +24,14 @@ module Gitlab def transmit(event) transmit_counter.increment + + if event.payload.present? + channel = event.payload[:channel_class] + operation = operation_name_from(event.payload) + data_size = ::ActiveSupport::JSON.encode(event.payload[:data]).bytesize + + transmitted_bytes_histogram.observe({ channel: channel, operation: operation }, data_size) + end end def broadcast(event) @@ -31,6 +40,13 @@ module Gitlab private + # When possible tries to query operation name + def operation_name_from(payload) + data = payload.dig(:data, 'result', 'data') || {} + + data.each_key.first + end + def transmit_counter strong_memoize("transmission_counter") do ::Gitlab::Metrics.counter( @@ -66,6 +82,12 @@ module Gitlab ) end end + + def transmitted_bytes_histogram + strong_memoize("transmitted_bytes_histogram") do + ::Gitlab::Metrics.histogram(DATA_TRANSMITTED_BYTES, 'Message size, in bytes, transmitted over action cable') + end + end end end end diff --git a/lib/gitlab/metrics/subscribers/active_record.rb b/lib/gitlab/metrics/subscribers/active_record.rb index 9f7884e1364..a8fcad9ff9f 100644 --- a/lib/gitlab/metrics/subscribers/active_record.rb +++ b/lib/gitlab/metrics/subscribers/active_record.rb @@ -15,8 +15,8 @@ module Gitlab TRANSACTION_DURATION_BUCKET = [0.1, 0.25, 1].freeze DB_LOAD_BALANCING_COUNTERS = %i{ - db_replica_count db_replica_cached_count db_replica_wal_count - db_primary_count db_primary_cached_count db_primary_wal_count + db_replica_count db_replica_cached_count db_replica_wal_count db_replica_wal_cached_count + db_primary_count db_primary_cached_count db_primary_wal_count db_primary_wal_cached_count }.freeze DB_LOAD_BALANCING_DURATIONS = %i{db_primary_duration_s db_replica_duration_s}.freeze @@ -72,6 +72,14 @@ module Gitlab DB_LOAD_BALANCING_DURATIONS.each do |duration| payload[duration] = ::Gitlab::SafeRequestStore[duration].to_f.round(3) end + + if Feature.enabled?(:multiple_database_metrics, default_enabled: :yaml) + ::Gitlab::SafeRequestStore[:duration_by_database]&.each do |dbname, duration_by_role| + duration_by_role.each do |db_role, duration| + payload[:"db_#{db_role}_#{dbname}_duration_s"] = duration.to_f.round(3) + end + end + end end end end @@ -83,9 +91,14 @@ module Gitlab end def increment_db_role_counters(db_role, payload) + cached = cached_query?(payload) increment("db_#{db_role}_count".to_sym) - increment("db_#{db_role}_cached_count".to_sym) if cached_query?(payload) - increment("db_#{db_role}_wal_count".to_sym) if !cached_query?(payload) && wal_command?(payload) + increment("db_#{db_role}_cached_count".to_sym) if cached + + if wal_command?(payload) + increment("db_#{db_role}_wal_count".to_sym) + increment("db_#{db_role}_wal_cached_count".to_sym) if cached + end end def observe_db_role_duration(db_role, event) @@ -93,9 +106,18 @@ module Gitlab buckets ::Gitlab::Metrics::Subscribers::ActiveRecord::SQL_DURATION_BUCKET end + return unless ::Gitlab::SafeRequestStore.active? + duration = event.duration / 1000.0 duration_key = "db_#{db_role}_duration_s".to_sym ::Gitlab::SafeRequestStore[duration_key] = (::Gitlab::SafeRequestStore[duration_key].presence || 0) + duration + + # Per database metrics + dbname = ::Gitlab::Database.dbname(event.payload[:connection]) + ::Gitlab::SafeRequestStore[:duration_by_database] ||= {} + ::Gitlab::SafeRequestStore[:duration_by_database][dbname] ||= {} + ::Gitlab::SafeRequestStore[:duration_by_database][dbname][db_role] ||= 0 + ::Gitlab::SafeRequestStore[:duration_by_database][dbname][db_role] += duration end def ignored_query?(payload) diff --git a/lib/gitlab/metrics/subscribers/load_balancing.rb b/lib/gitlab/metrics/subscribers/load_balancing.rb new file mode 100644 index 00000000000..333fc63ebef --- /dev/null +++ b/lib/gitlab/metrics/subscribers/load_balancing.rb @@ -0,0 +1,70 @@ +# frozen_string_literal: true + +module Gitlab + module Metrics + module Subscribers + class LoadBalancing < ActiveSupport::Subscriber + attach_to :load_balancing + + PROMETHEUS_COUNTER = :gitlab_transaction_caught_up_replica_pick_count_total + LOG_COUNTERS = { true => :caught_up_replica_pick_ok, false => :caught_up_replica_pick_fail }.freeze + + def caught_up_replica_pick(event) + return unless Gitlab::SafeRequestStore.active? && ::Gitlab::Database::LoadBalancing.enable? + + result = event.payload[:result] + counter_name = counter(result) + + increment(counter_name) + end + + # we want to update Prometheus counter after the controller/action are set + def web_transaction_completed(_event) + return unless Gitlab::SafeRequestStore.active? && ::Gitlab::Database::LoadBalancing.enable? + + LOG_COUNTERS.keys.each { |result| increment_prometheus_for_result_label(result) } + end + + def self.load_balancing_payload + return {} unless Gitlab::SafeRequestStore.active? && ::Gitlab::Database::LoadBalancing.enable? + + {}.tap do |payload| + LOG_COUNTERS.values.each do |counter| + value = Gitlab::SafeRequestStore[counter] + + payload[counter] = value.to_i if value + end + end + end + + private + + def increment(counter) + Gitlab::SafeRequestStore[counter] = Gitlab::SafeRequestStore[counter].to_i + 1 + end + + def increment_prometheus_for_result_label(label_value) + counter_name = counter(label_value) + return unless (counter_value = Gitlab::SafeRequestStore[counter_name]) + + increment_prometheus(labels: { result: label_value }, value: counter_value.to_i) + end + + def increment_prometheus(labels:, value:) + current_transaction&.increment(PROMETHEUS_COUNTER, value, labels) do + docstring 'Caught up replica pick result' + label_keys labels.keys + end + end + + def counter(result) + LOG_COUNTERS[result] + end + + def current_transaction + ::Gitlab::Metrics::WebTransaction.current + end + end + end + end +end |