diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2021-12-08 18:13:43 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2021-12-08 18:13:43 +0300 |
commit | 26b7ec2c6a7ffb01850dbecb0930b4f9412fe0f2 (patch) | |
tree | c9c8fdd391f97df8eea503a4ff21bb0b31ecb127 /sidekiq_cluster/cli.rb | |
parent | 886ecba0bd2d964504b43303a39cfa2386f0feed (diff) |
Add latest changes from gitlab-org/gitlab@master
Diffstat (limited to 'sidekiq_cluster/cli.rb')
-rw-r--r-- | sidekiq_cluster/cli.rb | 57 |
1 files changed, 57 insertions, 0 deletions
diff --git a/sidekiq_cluster/cli.rb b/sidekiq_cluster/cli.rb index 2dbb1e9c7c7..274b7c03e14 100644 --- a/sidekiq_cluster/cli.rb +++ b/sidekiq_cluster/cli.rb @@ -1,5 +1,7 @@ # frozen_string_literal: true +require_relative '../config/bundler_setup' + require 'optparse' require 'logger' require 'time' @@ -12,6 +14,7 @@ require_relative '../lib/gitlab/sidekiq_config/cli_methods' require_relative '../lib/gitlab/sidekiq_config/worker_matcher' require_relative '../lib/gitlab/sidekiq_logging/json_formatter' require_relative '../lib/gitlab/process_management' +require_relative '../metrics_server/metrics_server' require_relative 'sidekiq_cluster' module Gitlab @@ -89,6 +92,8 @@ module Gitlab @logger.info("Starting cluster with #{queue_groups.length} processes") end + start_metrics_server(wipe_metrics_dir: true) + @processes = SidekiqCluster.start( queue_groups, env: @environment, @@ -154,17 +159,69 @@ module Gitlab while @alive sleep(@interval) + if metrics_server_enabled? && ProcessManagement.process_died?(@metrics_server_pid) + @logger.warn('Metrics server went away') + start_metrics_server(wipe_metrics_dir: false) + end + unless ProcessManagement.all_alive?(@processes) # If a child process died we'll just terminate the whole cluster. It's up to # runit and such to then restart the cluster. @logger.info('A worker terminated, shutting down the cluster') + stop_metrics_server ProcessManagement.signal_processes(@processes, :TERM) break end end end + def start_metrics_server(wipe_metrics_dir: false) + return unless metrics_server_enabled? + + @logger.info("Starting metrics server on port #{sidekiq_exporter_port}") + @metrics_server_pid = MetricsServer.spawn('sidekiq', wipe_metrics_dir: wipe_metrics_dir) + end + + def sidekiq_exporter_enabled? + ::Settings.monitoring.sidekiq_exporter.enabled + rescue Settingslogic::MissingSetting + nil + end + + def exporter_has_a_unique_port? + # In https://gitlab.com/gitlab-org/gitlab/-/issues/345802 we added settings for sidekiq_health_checks. + # These settings default to the same values as sidekiq_exporter for backwards compatibility. + # If a different port for sidekiq_health_checks has been set up, we know that the + # user wants to serve health checks and metrics from different servers. + return false if sidekiq_health_check_port.nil? || sidekiq_exporter_port.nil? + + sidekiq_exporter_port != sidekiq_health_check_port + end + + def sidekiq_exporter_port + ::Settings.monitoring.sidekiq_exporter.port + rescue Settingslogic::MissingSetting + nil + end + + def sidekiq_health_check_port + ::Settings.monitoring.sidekiq_health_checks.port + rescue Settingslogic::MissingSetting + nil + end + + def metrics_server_enabled? + !@dryrun && sidekiq_exporter_enabled? && exporter_has_a_unique_port? + end + + def stop_metrics_server + return unless @metrics_server_pid + + @logger.info("Stopping metrics server (PID #{@metrics_server_pid})") + ProcessManagement.signal(@metrics_server_pid, :TERM) + end + def option_parser OptionParser.new do |opt| opt.banner = "#{File.basename(__FILE__)} [QUEUE,QUEUE] [QUEUE] ... [OPTIONS]" |