Add latest changes from gitlab-org/gitlab@14-6-stable-eev14.6.0-rc42

5 files changed, 59 insertions, 50 deletions

diff --git a/lib/gitlab/metrics/exporter/base_exporter.rb b/lib/gitlab/metrics/exporter/base_exporter.rb
index ff8b8bf2237..47c862c0232 100644
--- a/lib/gitlab/metrics/exporter/base_exporter.rb
+++ b/lib/gitlab/metrics/exporter/base_exporter.rb
@@ -11,12 +11,14 @@ module Gitlab
 
         attr_accessor :readiness_checks
 
-        def enabled?
-          settings.enabled
+        def initialize(settings, **options)
+          super(**options)
+
+          @settings = settings
         end
 
-        def settings
-          raise NotImplementedError
+        def enabled?
+          settings.enabled
         end
 
         def log_filename
@@ -25,6 +27,8 @@ module Gitlab
 
         private
 
+        attr_reader :settings
+
         def start_working
           logger = WEBrick::Log.new(log_filename)
           logger.time_format = "[%Y-%m-%dT%H:%M:%S.%L%z]"
diff --git a/lib/gitlab/metrics/exporter/sidekiq_exporter.rb b/lib/gitlab/metrics/exporter/sidekiq_exporter.rb
index 4d38d9e67bf..eea71fda6a0 100644
--- a/lib/gitlab/metrics/exporter/sidekiq_exporter.rb
+++ b/lib/gitlab/metrics/exporter/sidekiq_exporter.rb
@@ -4,10 +4,6 @@ module Gitlab
   module Metrics
     module Exporter
       class SidekiqExporter < BaseExporter
-        def settings
-          Settings.monitoring.sidekiq_exporter
-        end
-
         def log_filename
           if settings['log_enabled']
             File.join(Rails.root, 'log', 'sidekiq_exporter.log')
@@ -15,29 +11,6 @@ module Gitlab
             File::NULL
           end
         end
-
-        private
-
-        # Sidekiq Exporter does not work properly in sidekiq-cluster
-        # mode. It tries to start the service on the same port for
-        # each of the cluster workers, this results in failure
-        # due to duplicate binding.
-        #
-        # For now we ignore this error, as metrics are still "kind of"
-        # valid as they are rendered from shared directory.
-        #
-        # Issue: https://gitlab.com/gitlab-org/gitlab/issues/5714
-        def start_working
-          super
-        rescue Errno::EADDRINUSE => e
-          Sidekiq.logger.error(
-            class: self.class.to_s,
-            message: 'Cannot start sidekiq_exporter',
-            'exception.message' => e.message
-          )
-
-          false
-        end
       end
     end
   end
diff --git a/lib/gitlab/metrics/exporter/web_exporter.rb b/lib/gitlab/metrics/exporter/web_exporter.rb
index c5fa1e545d7..d41484aaaa7 100644
--- a/lib/gitlab/metrics/exporter/web_exporter.rb
+++ b/lib/gitlab/metrics/exporter/web_exporter.rb
@@ -27,7 +27,7 @@ module Gitlab
 
         # This exporter is always run on master process
         def initialize
-          super
+          super(Settings.monitoring.web_exporter)
 
           # DEPRECATED:
           # these `readiness_checks` are deprecated
@@ -39,10 +39,6 @@ module Gitlab
           ]
         end
 
-        def settings
-          Gitlab.config.monitoring.web_exporter
-        end
-
         def log_filename
           File.join(Rails.root, 'log', 'web_exporter.log')
         end
diff --git a/lib/gitlab/metrics/samplers/database_sampler.rb b/lib/gitlab/metrics/samplers/database_sampler.rb
index 5d7f434b660..965d85e20e5 100644
--- a/lib/gitlab/metrics/samplers/database_sampler.rb
+++ b/lib/gitlab/metrics/samplers/database_sampler.rb
@@ -38,16 +38,42 @@ module Gitlab
         end
 
         def host_stats
-          return [] unless ActiveRecord::Base.connected?
+          connection_class_stats + replica_host_stats
+        end
+
+        def connection_class_stats
+          Gitlab::Database.database_base_models.each_value.with_object([]) do |base_model, stats|
+            next unless base_model.connected?
+
+            stats << { labels: labels_for_class(base_model), stats: base_model.connection_pool.stat }
+          end
+        end
+
+        def replica_host_stats
+          Gitlab::Database::LoadBalancing.each_load_balancer.with_object([]) do |load_balancer, stats|
+            next if load_balancer.primary_only?
 
-          [{ labels: labels_for_class(ActiveRecord::Base), stats: ActiveRecord::Base.connection_pool.stat }]
+            load_balancer.host_list.hosts.each do |host|
+              stats << { labels: labels_for_replica_host(load_balancer, host), stats: host.connection.pool.stat }
+            end
+          end
         end
 
         def labels_for_class(klass)
           {
             host: klass.connection_db_config.host,
             port: klass.connection_db_config.configuration_hash[:port],
-            class: klass.to_s
+            class: klass.to_s,
+            db_config_name: klass.connection_db_config.name
+          }
+        end
+
+        def labels_for_replica_host(load_balancer, host)
+          {
+            host: host.host,
+            port: host.port,
+            class: load_balancer.configuration.primary_connection_specification_name,
+            db_config_name: Gitlab::Database.db_config_name(host.connection)
           }
         end
       end
diff --git a/lib/gitlab/metrics/subscribers/active_record.rb b/lib/gitlab/metrics/subscribers/active_record.rb
index df0582149a9..715dd86d93c 100644
--- a/lib/gitlab/metrics/subscribers/active_record.rb
+++ b/lib/gitlab/metrics/subscribers/active_record.rb
@@ -5,6 +5,8 @@ module Gitlab
     module Subscribers
       # Class for tracking the total query duration of a transaction.
       class ActiveRecord < ActiveSupport::Subscriber
+        extend Gitlab::Utils::StrongMemoize
+
         attach_to :active_record
 
         IGNORABLE_SQL = %w{BEGIN COMMIT}.freeze
@@ -107,7 +109,7 @@ module Gitlab
 
           # Per database metrics
           db_config_name = db_config_name(event.payload)
-          duration_key = compose_metric_key(:duration_s, db_role, db_config_name)
+          duration_key = compose_metric_key(:duration_s, nil, db_config_name)
           ::Gitlab::SafeRequestStore[duration_key] = (::Gitlab::SafeRequestStore[duration_key].presence || 0) + duration
         end
 
@@ -144,7 +146,7 @@ module Gitlab
           # when we are also logging the db_role. Otherwise it will be hard to
           # tell if the log key is referring to a db_role OR a db_config_name.
           if db_role.present? && db_config_name.present?
-            log_key = compose_metric_key(counter, db_role, db_config_name)
+            log_key = compose_metric_key(counter, nil, db_config_name)
             Gitlab::SafeRequestStore[log_key] = Gitlab::SafeRequestStore[log_key].to_i + 1
           end
         end
@@ -172,26 +174,34 @@ module Gitlab
         end
 
         def self.load_balancing_metric_counter_keys
-          load_balancing_metric_keys(DB_LOAD_BALANCING_COUNTERS)
+          strong_memoize(:load_balancing_metric_counter_keys) do
+            load_balancing_metric_keys(DB_LOAD_BALANCING_COUNTERS)
+          end
         end
 
         def self.load_balancing_metric_duration_keys
-          load_balancing_metric_keys(DB_LOAD_BALANCING_DURATIONS)
+          strong_memoize(:load_balancing_metric_duration_keys) do
+            load_balancing_metric_keys(DB_LOAD_BALANCING_DURATIONS)
+          end
         end
 
         def self.load_balancing_metric_keys(metrics)
-          [].tap do |counters|
+          counters = []
+
+          metrics.each do |metric|
             DB_LOAD_BALANCING_ROLES.each do |role|
-              metrics.each do |metric|
-                counters << compose_metric_key(metric, role)
-                next unless ENV['GITLAB_MULTIPLE_DATABASE_METRICS']
+              counters << compose_metric_key(metric, role)
+            end
 
-                ::Gitlab::Database.db_config_names.each do |config_name|
-                  counters << compose_metric_key(metric, role, config_name)
-                end
+            if ENV['GITLAB_MULTIPLE_DATABASE_METRICS']
+              ::Gitlab::Database.db_config_names.each do |config_name|
+                counters << compose_metric_key(metric, nil, config_name) # main
+                counters << compose_metric_key(metric, nil, config_name + ::Gitlab::Database::LoadBalancing::LoadBalancer::REPLICA_SUFFIX) # main_replica
               end
             end
           end
+
+          counters
         end
 
         def compose_metric_key(metric, db_role = nil, db_config_name = nil)