Add latest changes from gitlab-org/gitlab@14-7-stable-eev14.7.0-rc42

9 files changed, 138 insertions, 43 deletions

diff --git a/lib/gitlab/metrics/exporter/base_exporter.rb b/lib/gitlab/metrics/exporter/base_exporter.rb
index 47c862c0232..190d3d3fd2f 100644
--- a/lib/gitlab/metrics/exporter/base_exporter.rb
+++ b/lib/gitlab/metrics/exporter/base_exporter.rb
@@ -11,44 +11,41 @@ module Gitlab
 
         attr_accessor :readiness_checks
 
-        def initialize(settings, **options)
+        def initialize(settings, log_enabled:, log_file:, gc_requests: false, **options)
           super(**options)
 
           @settings = settings
+          @gc_requests = gc_requests
+
+          # log_enabled does not exist for all exporters
+          log_sink = log_enabled ? File.join(Rails.root, 'log', log_file) : File::NULL
+          @logger = WEBrick::Log.new(log_sink)
+          @logger.time_format = "[%Y-%m-%dT%H:%M:%S.%L%z]"
         end
 
         def enabled?
           settings.enabled
         end
 
-        def log_filename
-          raise NotImplementedError
-        end
-
         private
 
-        attr_reader :settings
+        attr_reader :settings, :logger
 
         def start_working
-          logger = WEBrick::Log.new(log_filename)
-          logger.time_format = "[%Y-%m-%dT%H:%M:%S.%L%z]"
-
           access_log = [
             [logger, WEBrick::AccessLog::COMBINED_LOG_FORMAT]
           ]
 
           @server = ::WEBrick::HTTPServer.new(
             Port: settings.port, BindAddress: settings.address,
-            Logger: logger, AccessLog: access_log)
-          server.mount_proc '/readiness' do |req, res|
-            render_probe(readiness_probe, req, res)
-          end
-          server.mount_proc '/liveness' do |req, res|
-            render_probe(liveness_probe, req, res)
-          end
+            Logger: logger, AccessLog: access_log
+          )
           server.mount '/', Rack::Handler::WEBrick, rack_app
 
           true
+        rescue StandardError => e
+          logger.error(e)
+          false
         end
 
         def run_thread
@@ -72,8 +69,16 @@ module Gitlab
         end
 
         def rack_app
+          readiness = readiness_probe
+          liveness = liveness_probe
+          pid = thread_name
+          gc_requests = @gc_requests
+
           Rack::Builder.app do
             use Rack::Deflater
+            use Gitlab::Metrics::Exporter::MetricsMiddleware, pid
+            use Gitlab::Metrics::Exporter::HealthChecksMiddleware, readiness, liveness
+            use Gitlab::Metrics::Exporter::GcRequestMiddleware if gc_requests
             use ::Prometheus::Client::Rack::Exporter if ::Gitlab::Metrics.metrics_folder_present?
             run -> (env) { [404, {}, ['']] }
           end
@@ -86,14 +91,6 @@ module Gitlab
         def liveness_probe
           ::Gitlab::HealthChecks::Probes::Collection.new
         end
-
-        def render_probe(probe, req, res)
-          result = probe.execute
-
-          res.status = result.http_status
-          res.content_type = 'application/json; charset=utf-8'
-          res.body = result.json.to_json
-        end
       end
     end
   end
diff --git a/lib/gitlab/metrics/exporter/gc_request_middleware.rb b/lib/gitlab/metrics/exporter/gc_request_middleware.rb
new file mode 100644
index 00000000000..3806b0e2bd1
--- /dev/null
+++ b/lib/gitlab/metrics/exporter/gc_request_middleware.rb
@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Metrics
+    module Exporter
+      class GcRequestMiddleware
+        def initialize(app)
+          @app = app
+        end
+
+        def call(env)
+          @app.call(env).tap do
+            GC.start
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/metrics/exporter/health_checks_middleware.rb b/lib/gitlab/metrics/exporter/health_checks_middleware.rb
new file mode 100644
index 00000000000..c43b8004b72
--- /dev/null
+++ b/lib/gitlab/metrics/exporter/health_checks_middleware.rb
@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Metrics
+    module Exporter
+      class HealthChecksMiddleware
+        def initialize(app, readiness_probe, liveness_probe)
+          @app = app
+          @readiness_probe = readiness_probe
+          @liveness_probe = liveness_probe
+        end
+
+        def call(env)
+          case env['PATH_INFO']
+          when '/readiness' then render_probe(@readiness_probe)
+          when '/liveness' then render_probe(@liveness_probe)
+          else @app.call(env)
+          end
+        end
+
+        private
+
+        def render_probe(probe)
+          result = probe.execute
+
+          [
+            result.http_status,
+            { 'Content-Type' => 'application/json; charset=utf-8' },
+            [result.json.to_json]
+          ]
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/metrics/exporter/metrics_middleware.rb b/lib/gitlab/metrics/exporter/metrics_middleware.rb
new file mode 100644
index 00000000000..e17f1c13cf0
--- /dev/null
+++ b/lib/gitlab/metrics/exporter/metrics_middleware.rb
@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Metrics
+    module Exporter
+      class MetricsMiddleware
+        def initialize(app, pid)
+          @app = app
+          default_labels = {
+            pid: pid
+          }
+          @requests_total = Gitlab::Metrics.counter(
+            :exporter_http_requests_total, 'Total number of HTTP requests', default_labels
+          )
+          @request_durations = Gitlab::Metrics.histogram(
+            :exporter_http_request_duration_seconds,
+            'HTTP request duration histogram (seconds)',
+            default_labels,
+            [0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10]
+          )
+        end
+
+        def call(env)
+          start = Gitlab::Metrics::System.monotonic_time
+          @app.call(env).tap do |response|
+            duration = Gitlab::Metrics::System.monotonic_time - start
+
+            labels = {
+              method: env['REQUEST_METHOD'].downcase,
+              path:   env['PATH_INFO'].to_s,
+              code:   response.first.to_s
+            }
+
+            @requests_total.increment(labels)
+            @request_durations.observe(labels, duration)
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/metrics/exporter/sidekiq_exporter.rb b/lib/gitlab/metrics/exporter/sidekiq_exporter.rb
index eea71fda6a0..afecf6546f8 100644
--- a/lib/gitlab/metrics/exporter/sidekiq_exporter.rb
+++ b/lib/gitlab/metrics/exporter/sidekiq_exporter.rb
@@ -4,12 +4,11 @@ module Gitlab
   module Metrics
     module Exporter
       class SidekiqExporter < BaseExporter
-        def log_filename
-          if settings['log_enabled']
-            File.join(Rails.root, 'log', 'sidekiq_exporter.log')
-          else
-            File::NULL
-          end
+        def initialize(settings, **options)
+          super(settings,
+            log_enabled: settings['log_enabled'],
+            log_file: 'sidekiq_exporter.log',
+            **options)
         end
       end
     end
diff --git a/lib/gitlab/metrics/exporter/web_exporter.rb b/lib/gitlab/metrics/exporter/web_exporter.rb
index d41484aaaa7..c05ad8ccf42 100644
--- a/lib/gitlab/metrics/exporter/web_exporter.rb
+++ b/lib/gitlab/metrics/exporter/web_exporter.rb
@@ -26,8 +26,8 @@ module Gitlab
         attr_reader :running
 
         # This exporter is always run on master process
-        def initialize
-          super(Settings.monitoring.web_exporter)
+        def initialize(**options)
+          super(Settings.monitoring.web_exporter, log_enabled: true, log_file: 'web_exporter.log', **options)
 
           # DEPRECATED:
           # these `readiness_checks` are deprecated
@@ -39,10 +39,6 @@ module Gitlab
           ]
         end
 
-        def log_filename
-          File.join(Rails.root, 'log', 'web_exporter.log')
-        end
-
         def mark_as_not_running!
           @running = false
         end
diff --git a/lib/gitlab/metrics/samplers/action_cable_sampler.rb b/lib/gitlab/metrics/samplers/action_cable_sampler.rb
index adce3030d0d..1f50371cae9 100644
--- a/lib/gitlab/metrics/samplers/action_cable_sampler.rb
+++ b/lib/gitlab/metrics/samplers/action_cable_sampler.rb
@@ -6,8 +6,8 @@ module Gitlab
       class ActionCableSampler < BaseSampler
         DEFAULT_SAMPLING_INTERVAL_SECONDS = 5
 
-        def initialize(interval = nil, action_cable: ::ActionCable.server)
-          super(interval)
+        def initialize(action_cable: ::ActionCable.server, **options)
+          super(**options)
           @action_cable = action_cable
         end
 
diff --git a/lib/gitlab/metrics/samplers/base_sampler.rb b/lib/gitlab/metrics/samplers/base_sampler.rb
index 52d80c3c27e..b2a9de21145 100644
--- a/lib/gitlab/metrics/samplers/base_sampler.rb
+++ b/lib/gitlab/metrics/samplers/base_sampler.rb
@@ -9,7 +9,10 @@ module Gitlab
         attr_reader :interval
 
         # interval - The sampling interval in seconds.
-        def initialize(interval = nil)
+        # warmup   - When true, takes a single sample eagerly before entering the sampling loop.
+        #            This can be useful to ensure that all metrics files exist after `start` returns,
+        #            since prometheus-client-mmap creates them lazily upon first access.
+        def initialize(interval: nil, logger: Logger.new($stdout), warmup: false, **options)
           interval ||= ENV[interval_env_key]&.to_i
           interval ||= self.class::DEFAULT_SAMPLING_INTERVAL_SECONDS
           interval_half = interval.to_f / 2
@@ -17,13 +20,16 @@ module Gitlab
           @interval = interval
           @interval_steps = (-interval_half..interval_half).step(0.1).to_a
 
-          super()
+          @logger = logger
+          @warmup = warmup
+
+          super(**options)
         end
 
         def safe_sample
           sample
         rescue StandardError => e
-          ::Gitlab::AppLogger.warn("#{self.class}: #{e}, stopping")
+          @logger.warn("#{self.class}: #{e}, stopping")
           stop
         end
 
@@ -63,6 +69,8 @@ module Gitlab
         def start_working
           @running = true
 
+          safe_sample if @warmup
+
           true
         end
 
diff --git a/lib/gitlab/metrics/samplers/ruby_sampler.rb b/lib/gitlab/metrics/samplers/ruby_sampler.rb
index b1c5e9800da..d71ee671b8d 100644
--- a/lib/gitlab/metrics/samplers/ruby_sampler.rb
+++ b/lib/gitlab/metrics/samplers/ruby_sampler.rb
@@ -7,12 +7,12 @@ module Gitlab
         DEFAULT_SAMPLING_INTERVAL_SECONDS = 60
         GC_REPORT_BUCKETS = [0.01, 0.05, 0.1, 0.2, 0.3, 0.5, 1].freeze
 
-        def initialize(*)
+        def initialize(...)
           GC::Profiler.clear
 
           metrics[:process_start_time_seconds].set(labels, Time.now.to_i)
 
-          super
+          super(...)
         end
 
         def metrics