diff options
Diffstat (limited to 'lib/gitlab/metrics')
| -rw-r--r-- | lib/gitlab/metrics/requests_rack_middleware.rb | 16 | ||||
| -rw-r--r-- | lib/gitlab/metrics/samplers/base_sampler.rb | 2 | ||||
| -rw-r--r-- | lib/gitlab/metrics/subscribers/action_cable.rb | 2 | ||||
| -rw-r--r-- | lib/gitlab/metrics/subscribers/action_view.rb | 2 | ||||
| -rw-r--r-- | lib/gitlab/metrics/subscribers/active_record.rb | 130 | ||||
| -rw-r--r-- | lib/gitlab/metrics/subscribers/rails_cache.rb | 2 |
6 files changed, 106 insertions, 48 deletions
diff --git a/lib/gitlab/metrics/requests_rack_middleware.rb b/lib/gitlab/metrics/requests_rack_middleware.rb index b99261b5c4d..6ba336d37cd 100644 --- a/lib/gitlab/metrics/requests_rack_middleware.rb +++ b/lib/gitlab/metrics/requests_rack_middleware.rb @@ -13,7 +13,7 @@ module Gitlab "put" => %w(200 202 204 400 401 403 404 405 406 409 410 422 500) }.freeze - HEALTH_ENDPOINT = /^\/-\/(liveness|readiness|health|metrics)\/?$/.freeze + HEALTH_ENDPOINT = %r{^/-/(liveness|readiness|health|metrics)/?$}.freeze FEATURE_CATEGORY_DEFAULT = 'unknown' @@ -66,28 +66,28 @@ module Gitlab def call(env) method = env['REQUEST_METHOD'].downcase method = 'INVALID' unless HTTP_METHODS.key?(method) - started = Gitlab::Metrics::System.monotonic_time + started = ::Gitlab::Metrics::System.monotonic_time health_endpoint = health_endpoint?(env['PATH_INFO']) status = 'undefined' begin status, headers, body = @app.call(env) - elapsed = Gitlab::Metrics::System.monotonic_time - started + elapsed = ::Gitlab::Metrics::System.monotonic_time - started - if !health_endpoint && Gitlab::Metrics.record_duration_for_status?(status) - RequestsRackMiddleware.http_request_duration_seconds.observe({ method: method }, elapsed) + if !health_endpoint && ::Gitlab::Metrics.record_duration_for_status?(status) + self.class.http_request_duration_seconds.observe({ method: method }, elapsed) end [status, headers, body] rescue StandardError - RequestsRackMiddleware.rack_uncaught_errors_count.increment + self.class.rack_uncaught_errors_count.increment raise ensure if health_endpoint - RequestsRackMiddleware.http_health_requests_total.increment(status: status.to_s, method: method) + self.class.http_health_requests_total.increment(status: status.to_s, method: method) else - RequestsRackMiddleware.http_requests_total.increment( + self.class.http_requests_total.increment( status: status.to_s, method: method, feature_category: feature_category.presence || FEATURE_CATEGORY_DEFAULT diff --git a/lib/gitlab/metrics/samplers/base_sampler.rb b/lib/gitlab/metrics/samplers/base_sampler.rb index 258aa93be38..52d80c3c27e 100644 --- a/lib/gitlab/metrics/samplers/base_sampler.rb +++ b/lib/gitlab/metrics/samplers/base_sampler.rb @@ -23,7 +23,7 @@ module Gitlab def safe_sample sample rescue StandardError => e - Gitlab::AppLogger.warn("#{self.class}: #{e}, stopping") + ::Gitlab::AppLogger.warn("#{self.class}: #{e}, stopping") stop end diff --git a/lib/gitlab/metrics/subscribers/action_cable.rb b/lib/gitlab/metrics/subscribers/action_cable.rb index 631b9209f14..9f955dfe79f 100644 --- a/lib/gitlab/metrics/subscribers/action_cable.rb +++ b/lib/gitlab/metrics/subscribers/action_cable.rb @@ -28,7 +28,7 @@ module Gitlab if event.payload.present? channel = event.payload[:channel_class] operation = operation_name_from(event.payload) - data_size = ::ActiveSupport::JSON.encode(event.payload[:data]).bytesize + data_size = Gitlab::Json.generate(event.payload[:data]).bytesize transmitted_bytes_histogram.observe({ channel: channel, operation: operation }, data_size) end diff --git a/lib/gitlab/metrics/subscribers/action_view.rb b/lib/gitlab/metrics/subscribers/action_view.rb index e1f1f37c905..fa129025bfe 100644 --- a/lib/gitlab/metrics/subscribers/action_view.rb +++ b/lib/gitlab/metrics/subscribers/action_view.rb @@ -40,7 +40,7 @@ module Gitlab end def current_transaction - Transaction.current + ::Gitlab::Metrics::Transaction.current end end end diff --git a/lib/gitlab/metrics/subscribers/active_record.rb b/lib/gitlab/metrics/subscribers/active_record.rb index a8fcad9ff9f..59b2f88041f 100644 --- a/lib/gitlab/metrics/subscribers/active_record.rb +++ b/lib/gitlab/metrics/subscribers/active_record.rb @@ -8,17 +8,15 @@ module Gitlab attach_to :active_record IGNORABLE_SQL = %w{BEGIN COMMIT}.freeze - DB_COUNTERS = %i{db_count db_write_count db_cached_count}.freeze - SQL_COMMANDS_WITH_COMMENTS_REGEX = /\A(\/\*.*\*\/\s)?((?!(.*[^\w'"](DELETE|UPDATE|INSERT INTO)[^\w'"])))(WITH.*)?(SELECT)((?!(FOR UPDATE|FOR SHARE)).)*$/i.freeze + DB_COUNTERS = %i{count write_count cached_count}.freeze + SQL_COMMANDS_WITH_COMMENTS_REGEX = %r{\A(/\*.*\*/\s)?((?!(.*[^\w'"](DELETE|UPDATE|INSERT INTO)[^\w'"])))(WITH.*)?(SELECT)((?!(FOR UPDATE|FOR SHARE)).)*$}i.freeze SQL_DURATION_BUCKET = [0.05, 0.1, 0.25].freeze TRANSACTION_DURATION_BUCKET = [0.1, 0.25, 1].freeze - DB_LOAD_BALANCING_COUNTERS = %i{ - db_replica_count db_replica_cached_count db_replica_wal_count db_replica_wal_cached_count - db_primary_count db_primary_cached_count db_primary_wal_count db_primary_wal_cached_count - }.freeze - DB_LOAD_BALANCING_DURATIONS = %i{db_primary_duration_s db_replica_duration_s}.freeze + DB_LOAD_BALANCING_ROLES = %i{replica primary}.freeze + DB_LOAD_BALANCING_COUNTERS = %i{count cached_count wal_count wal_cached_count}.freeze + DB_LOAD_BALANCING_DURATIONS = %i{duration_s}.freeze SQL_WAL_LOCATION_REGEX = /(pg_current_wal_insert_lsn\(\)::text|pg_last_wal_replay_lsn\(\)::text)/.freeze @@ -40,9 +38,10 @@ module Gitlab payload = event.payload return if ignored_query?(payload) - increment(:db_count) - increment(:db_cached_count) if cached_query?(payload) - increment(:db_write_count) unless select_sql_command?(payload) + db_config_name = db_config_name(event.payload) + increment(:count, db_config_name: db_config_name) + increment(:cached_count, db_config_name: db_config_name) if cached_query?(payload) + increment(:write_count, db_config_name: db_config_name) unless select_sql_command?(payload) observe(:gitlab_sql_duration_seconds, event) do buckets SQL_DURATION_BUCKET @@ -61,24 +60,17 @@ module Gitlab return {} unless Gitlab::SafeRequestStore.active? {}.tap do |payload| - DB_COUNTERS.each do |counter| - payload[counter] = Gitlab::SafeRequestStore[counter].to_i + db_counter_keys.each do |key| + payload[key] = Gitlab::SafeRequestStore[key].to_i end if ::Gitlab::SafeRequestStore.active? && ::Gitlab::Database::LoadBalancing.enable? - DB_LOAD_BALANCING_COUNTERS.each do |counter| + load_balancing_metric_counter_keys.each do |counter| payload[counter] = ::Gitlab::SafeRequestStore[counter].to_i end - DB_LOAD_BALANCING_DURATIONS.each do |duration| - payload[duration] = ::Gitlab::SafeRequestStore[duration].to_f.round(3) - end - if Feature.enabled?(:multiple_database_metrics, default_enabled: :yaml) - ::Gitlab::SafeRequestStore[:duration_by_database]&.each do |dbname, duration_by_role| - duration_by_role.each do |db_role, duration| - payload[:"db_#{db_role}_#{dbname}_duration_s"] = duration.to_f.round(3) - end - end + load_balancing_metric_duration_keys.each do |duration| + payload[duration] = ::Gitlab::SafeRequestStore[duration].to_f.round(3) end end end @@ -92,12 +84,15 @@ module Gitlab def increment_db_role_counters(db_role, payload) cached = cached_query?(payload) - increment("db_#{db_role}_count".to_sym) - increment("db_#{db_role}_cached_count".to_sym) if cached + + db_config_name = db_config_name(payload) + + increment(:count, db_role: db_role, db_config_name: db_config_name) + increment(:cached_count, db_role: db_role, db_config_name: db_config_name) if cached if wal_command?(payload) - increment("db_#{db_role}_wal_count".to_sym) - increment("db_#{db_role}_wal_cached_count".to_sym) if cached + increment(:wal_count, db_role: db_role, db_config_name: db_config_name) + increment(:wal_cached_count, db_role: db_role, db_config_name: db_config_name) if cached end end @@ -109,15 +104,13 @@ module Gitlab return unless ::Gitlab::SafeRequestStore.active? duration = event.duration / 1000.0 - duration_key = "db_#{db_role}_duration_s".to_sym + duration_key = compose_metric_key(:duration_s, db_role) ::Gitlab::SafeRequestStore[duration_key] = (::Gitlab::SafeRequestStore[duration_key].presence || 0) + duration # Per database metrics - dbname = ::Gitlab::Database.dbname(event.payload[:connection]) - ::Gitlab::SafeRequestStore[:duration_by_database] ||= {} - ::Gitlab::SafeRequestStore[:duration_by_database][dbname] ||= {} - ::Gitlab::SafeRequestStore[:duration_by_database][dbname][db_role] ||= 0 - ::Gitlab::SafeRequestStore[:duration_by_database][dbname][db_role] += duration + db_config_name = db_config_name(event.payload) + duration_key = compose_metric_key(:duration_s, db_role, db_config_name) + ::Gitlab::SafeRequestStore[duration_key] = (::Gitlab::SafeRequestStore[duration_key].presence || 0) + duration end def ignored_query?(payload) @@ -132,19 +125,84 @@ module Gitlab payload[:sql].match(SQL_COMMANDS_WITH_COMMENTS_REGEX) end - def increment(counter) - current_transaction&.increment("gitlab_transaction_#{counter}_total".to_sym, 1) + def increment(counter, db_config_name:, db_role: nil) + log_key = compose_metric_key(counter, db_role) - Gitlab::SafeRequestStore[counter] = Gitlab::SafeRequestStore[counter].to_i + 1 + prometheus_key = if db_role + :"gitlab_transaction_db_#{db_role}_#{counter}_total" + else + :"gitlab_transaction_db_#{counter}_total" + end + + if ENV['GITLAB_MULTIPLE_DATABASE_METRICS'] + current_transaction&.increment(prometheus_key, 1, { db_config_name: db_config_name }) + else + current_transaction&.increment(prometheus_key, 1) + end + + Gitlab::SafeRequestStore[log_key] = Gitlab::SafeRequestStore[log_key].to_i + 1 + + # To avoid confusing log keys we only log the db_config_name metrics + # when we are also logging the db_role. Otherwise it will be hard to + # tell if the log key is referring to a db_role OR a db_config_name. + if db_role.present? && db_config_name.present? + log_key = compose_metric_key(counter, db_role, db_config_name) + Gitlab::SafeRequestStore[log_key] = Gitlab::SafeRequestStore[log_key].to_i + 1 + end end def observe(histogram, event, &block) - current_transaction&.observe(histogram, event.duration / 1000.0, &block) + db_config_name = db_config_name(event.payload) + + if ENV['GITLAB_MULTIPLE_DATABASE_METRICS'] + current_transaction&.observe(histogram, event.duration / 1000.0, { db_config_name: db_config_name }, &block) + else + current_transaction&.observe(histogram, event.duration / 1000.0, &block) + end end def current_transaction ::Gitlab::Metrics::WebTransaction.current || ::Gitlab::Metrics::BackgroundTransaction.current end + + def db_config_name(payload) + ::Gitlab::Database.db_config_name(payload[:connection]) + end + + def self.db_counter_keys + DB_COUNTERS.map { |c| compose_metric_key(c) } + end + + def self.load_balancing_metric_counter_keys + load_balancing_metric_keys(DB_LOAD_BALANCING_COUNTERS) + end + + def self.load_balancing_metric_duration_keys + load_balancing_metric_keys(DB_LOAD_BALANCING_DURATIONS) + end + + def self.load_balancing_metric_keys(metrics) + [].tap do |counters| + DB_LOAD_BALANCING_ROLES.each do |role| + metrics.each do |metric| + counters << compose_metric_key(metric, role) + next unless ENV['GITLAB_MULTIPLE_DATABASE_METRICS'] + + ::Gitlab::Database.db_config_names.each do |config_name| + counters << compose_metric_key(metric, role, config_name) + end + end + end + end + end + + def compose_metric_key(metric, db_role = nil, db_config_name = nil) + self.class.compose_metric_key(metric, db_role, db_config_name) + end + + def self.compose_metric_key(metric, db_role = nil, db_config_name = nil) + [:db, db_role, db_config_name, metric].compact.join("_").to_sym + end end end end diff --git a/lib/gitlab/metrics/subscribers/rails_cache.rb b/lib/gitlab/metrics/subscribers/rails_cache.rb index b274d2b1079..45344e79796 100644 --- a/lib/gitlab/metrics/subscribers/rails_cache.rb +++ b/lib/gitlab/metrics/subscribers/rails_cache.rb @@ -65,7 +65,7 @@ module Gitlab private def current_transaction - Transaction.current + ::Gitlab::Metrics::Transaction.current end def metric_cache_operation_duration_seconds |