diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2019-11-07 18:06:33 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2019-11-07 18:06:33 +0300 |
commit | 90a06a20be61bb6d48d77746091492831153e075 (patch) | |
tree | bdba99289605f8b5acf12159d02aeb23f8690202 /lib | |
parent | 84a0e65ac88c7a3db86a0e4347606ba093490bef (diff) |
Add latest changes from gitlab-org/gitlab@master
Diffstat (limited to 'lib')
-rw-r--r-- | lib/gitlab/cluster/lifecycle_events.rb | 61 | ||||
-rw-r--r-- | lib/gitlab/health_checks/master_check.rb | 66 | ||||
-rw-r--r-- | lib/gitlab/metrics/exporter/web_exporter.rb | 25 |
3 files changed, 113 insertions, 39 deletions
diff --git a/lib/gitlab/cluster/lifecycle_events.rb b/lib/gitlab/cluster/lifecycle_events.rb index f931a94938f..2b3dc94fc5e 100644 --- a/lib/gitlab/cluster/lifecycle_events.rb +++ b/lib/gitlab/cluster/lifecycle_events.rb @@ -10,38 +10,39 @@ module Gitlab # # We have the following lifecycle events. # - # - on_master_start: + # - on_before_fork (on master process): # # Unicorn/Puma Cluster: This will be called exactly once, # on startup, before the workers are forked. This is # called in the PARENT/MASTER process. # - # Sidekiq/Puma Single: This is called immediately. + # Sidekiq/Puma Single: This is not called. # - # - on_before_fork: + # - on_master_start (on master process): # # Unicorn/Puma Cluster: This will be called exactly once, # on startup, before the workers are forked. This is # called in the PARENT/MASTER process. # - # Sidekiq/Puma Single: This is not called. + # Sidekiq/Puma Single: This is called immediately. # - # - on_worker_start: + # - on_before_blackout_period (on master process): # - # Unicorn/Puma Cluster: This is called in the worker process - # exactly once before processing requests. + # Unicorn/Puma Cluster: This will be called before a blackout + # period when performing graceful shutdown of master. + # This is called on `master` process. # - # Sidekiq/Puma Single: This is called immediately. + # Sidekiq/Puma Single: This is not called. # - # - on_before_graceful_shutdown: + # - on_before_graceful_shutdown (on master process): # # Unicorn/Puma Cluster: This will be called before a graceful - # shutdown of workers starts happening. + # shutdown of workers starts happening, but after blackout period. # This is called on `master` process. # # Sidekiq/Puma Single: This is not called. # - # - on_before_master_restart: + # - on_before_master_restart (on master process): # # Unicorn: This will be called before a new master is spun up. # This is called on forked master before `execve` to become @@ -53,6 +54,13 @@ module Gitlab # # Sidekiq/Puma Single: This is not called. # + # - on_worker_start (on worker process): + # + # Unicorn/Puma Cluster: This is called in the worker process + # exactly once before processing requests. + # + # Sidekiq/Puma Single: This is called immediately. + # # Blocks will be executed in the order in which they are registered. # class LifecycleEvents @@ -75,6 +83,12 @@ module Gitlab end # Read the config/initializers/cluster_events_before_phased_restart.rb + def on_before_blackout_period(&block) + # Defer block execution + (@master_blackout_period ||= []) << block + end + + # Read the config/initializers/cluster_events_before_phased_restart.rb def on_before_graceful_shutdown(&block) # Defer block execution (@master_graceful_shutdown ||= []) << block @@ -97,27 +111,24 @@ module Gitlab # Lifecycle integration methods (called from unicorn.rb, puma.rb, etc.) # def do_worker_start - @worker_start_hooks&.each do |block| - block.call - end + call(@worker_start_hooks) end def do_before_fork - @before_fork_hooks&.each do |block| - block.call - end + call(@before_fork_hooks) end def do_before_graceful_shutdown - @master_graceful_shutdown&.each do |block| - block.call - end + call(@master_blackout_period) + + blackout_seconds = ::Settings.shutdown.blackout_seconds.to_i + sleep(blackout_seconds) if blackout_seconds > 0 + + call(@master_graceful_shutdown) end def do_before_master_restart - @master_restart_hooks&.each do |block| - block.call - end + call(@master_restart_hooks) end # DEPRECATED @@ -132,6 +143,10 @@ module Gitlab private + def call(hooks) + hooks&.each(&:call) + end + def in_clustered_environment? # Sidekiq doesn't fork return false if Sidekiq.server? diff --git a/lib/gitlab/health_checks/master_check.rb b/lib/gitlab/health_checks/master_check.rb new file mode 100644 index 00000000000..057bce84ddd --- /dev/null +++ b/lib/gitlab/health_checks/master_check.rb @@ -0,0 +1,66 @@ +# frozen_string_literal: true + +module Gitlab + module HealthChecks + # This check is registered on master, + # and validated by worker + class MasterCheck + extend SimpleAbstractCheck + + class << self + def register_master + # when we fork, we pass the read pipe to child + # child can then react on whether the other end + # of pipe is still available + @pipe_read, @pipe_write = IO.pipe + end + + def finish_master + close_read + close_write + end + + def register_worker + # fork needs to close the pipe + close_write + end + + private + + def close_read + @pipe_read&.close + @pipe_read = nil + end + + def close_write + @pipe_write&.close + @pipe_write = nil + end + + def metric_prefix + 'master_check' + end + + def successful?(result) + result + end + + def check + # the lack of pipe is a legitimate failure of check + return false unless @pipe_read + + @pipe_read.read_nonblock(1) + + true + rescue IO::EAGAINWaitReadable + # if it is blocked, it means that the pipe is still open + # and there's no data waiting on it + true + rescue EOFError + # the pipe is closed + false + end + end + end + end +end diff --git a/lib/gitlab/metrics/exporter/web_exporter.rb b/lib/gitlab/metrics/exporter/web_exporter.rb index 3940f6fa155..b6a27d8556a 100644 --- a/lib/gitlab/metrics/exporter/web_exporter.rb +++ b/lib/gitlab/metrics/exporter/web_exporter.rb @@ -20,6 +20,10 @@ module Gitlab def initialize super + # DEPRECATED: + # these `readiness_checks` are deprecated + # as presenting no value in a way how we run + # application: https://gitlab.com/gitlab-org/gitlab/issues/35343 self.readiness_checks = [ WebExporter::ExporterCheck.new(self), Gitlab::HealthChecks::PumaCheck, @@ -35,6 +39,10 @@ module Gitlab File.join(Rails.root, 'log', 'web_exporter.log') end + def mark_as_not_running! + @running = false + end + private def start_working @@ -43,24 +51,9 @@ module Gitlab end def stop_working - @running = false - wait_in_blackout_period if server && thread.alive? + mark_as_not_running! super end - - def wait_in_blackout_period - return unless blackout_seconds > 0 - - @server.logger.info( - message: 'starting blackout...', - duration_s: blackout_seconds) - - sleep(blackout_seconds) - end - - def blackout_seconds - settings['blackout_seconds'].to_i - end end end end |