From 1065f8ce7a261dff5a3077be46405343141733df Mon Sep 17 00:00:00 2001 From: Andrew Newdigate Date: Sat, 20 Oct 2018 19:00:19 +0100 Subject: Add experimental support for Puma This allows us (and others) to test drive Puma without it affecting all users. Puma can be enabled by setting the environment variable "EXPERIMENTAL_PUMA" to a non empty value. --- config/initializers/7_prometheus_metrics.rb | 22 +++++++- config/initializers/8_metrics.rb | 4 +- config/initializers/active_record_lifecycle.rb | 23 ++++++++ config/initializers/macos.rb | 13 +++++ config/initializers/rbtrace.rb | 9 +++ config/initializers/sidekiq.rb | 2 - config/puma.example.development.rb | 77 ++++++++++++++++++++++++++ config/unicorn.rb.example | 37 +++---------- config/unicorn.rb.example.development | 67 +++++++++++++++------- 9 files changed, 200 insertions(+), 54 deletions(-) create mode 100644 config/initializers/active_record_lifecycle.rb create mode 100644 config/initializers/macos.rb create mode 100644 config/initializers/rbtrace.rb create mode 100644 config/puma.example.development.rb (limited to 'config') diff --git a/config/initializers/7_prometheus_metrics.rb b/config/initializers/7_prometheus_metrics.rb index 146c4b1e024..8052880cc3d 100644 --- a/config/initializers/7_prometheus_metrics.rb +++ b/config/initializers/7_prometheus_metrics.rb @@ -26,9 +26,25 @@ Sidekiq.configure_server do |config| end if !Rails.env.test? && Gitlab::Metrics.prometheus_metrics_enabled? - unless Sidekiq.server? - Gitlab::Metrics::Samplers::UnicornSampler.initialize_instance(Settings.monitoring.unicorn_sampler_interval).start + Gitlab::Cluster::LifecycleEvents.on_worker_start do + defined?(::Prometheus::Client.reinitialize_on_pid_change) && Prometheus::Client.reinitialize_on_pid_change + + unless Sidekiq.server? + Gitlab::Metrics::Samplers::UnicornSampler.initialize_instance(Settings.monitoring.unicorn_sampler_interval).start + end + + Gitlab::Metrics::Samplers::RubySampler.initialize_instance(Settings.monitoring.ruby_sampler_interval).start end +end - Gitlab::Metrics::Samplers::RubySampler.initialize_instance(Settings.monitoring.ruby_sampler_interval).start +Gitlab::Cluster::LifecycleEvents.on_master_restart do + # The following is necessary to ensure stale Prometheus metrics don't + # accumulate over time. It needs to be done in this hook as opposed to + # inside an init script to ensure metrics files aren't deleted after new + # unicorn workers start after a SIGUSR2 is received. + prometheus_multiproc_dir = ENV['prometheus_multiproc_dir'] + if prometheus_multiproc_dir + old_metrics = Dir[File.join(prometheus_multiproc_dir, '*.db')] + FileUtils.rm_rf(old_metrics) + end end diff --git a/config/initializers/8_metrics.rb b/config/initializers/8_metrics.rb index eccf82ab8dc..c8d261d415e 100644 --- a/config/initializers/8_metrics.rb +++ b/config/initializers/8_metrics.rb @@ -158,7 +158,9 @@ if Gitlab::Metrics.enabled? && !Rails.env.test? GC::Profiler.enable - Gitlab::Metrics::Samplers::InfluxSampler.initialize_instance.start + Gitlab::Cluster::LifecycleEvents.on_worker_start do + Gitlab::Metrics::Samplers::InfluxSampler.initialize_instance.start + end module TrackNewRedisConnections def connect(*args) diff --git a/config/initializers/active_record_lifecycle.rb b/config/initializers/active_record_lifecycle.rb new file mode 100644 index 00000000000..7fa37121efc --- /dev/null +++ b/config/initializers/active_record_lifecycle.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +# Don't handle sidekiq configuration as it +# has its own special active record configuration here +if defined?(ActiveRecord::Base) && !Sidekiq.server? + Gitlab::Cluster::LifecycleEvents.on_worker_start do + ActiveSupport.on_load(:active_record) do + ActiveRecord::Base.establish_connection + + Rails.logger.debug("ActiveRecord connection established") + end + end +end + +if defined?(ActiveRecord::Base) + Gitlab::Cluster::LifecycleEvents.on_before_fork do + # the following is highly recommended for Rails + "preload_app true" + # as there's no need for the master process to hold a connection + ActiveRecord::Base.connection.disconnect! + + Rails.logger.debug("ActiveRecord connection disconnected") + end +end diff --git a/config/initializers/macos.rb b/config/initializers/macos.rb new file mode 100644 index 00000000000..f410af6ed47 --- /dev/null +++ b/config/initializers/macos.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +if /darwin/ =~ RUBY_PLATFORM + Gitlab::Cluster::LifecycleEvents.on_before_fork do + require 'fiddle' + + # Dynamically load Foundation.framework, ~implicitly~ initialising + # the Objective-C runtime before any forking happens in Unicorn + # + # From https://bugs.ruby-lang.org/issues/14009 + Fiddle.dlopen '/System/Library/Frameworks/Foundation.framework/Foundation' + end +end diff --git a/config/initializers/rbtrace.rb b/config/initializers/rbtrace.rb new file mode 100644 index 00000000000..6a1b71bf4bd --- /dev/null +++ b/config/initializers/rbtrace.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +if ENV['ENABLE_RBTRACE'] + Gitlab::Cluster::LifecycleEvents.on_worker_start do + # Unicorn clears out signals before it forks, so rbtrace won't work + # unless it is enabled after the fork. + require 'rbtrace' + end +end diff --git a/config/initializers/sidekiq.rb b/config/initializers/sidekiq.rb index bc6b7aed6aa..565efc858d1 100644 --- a/config/initializers/sidekiq.rb +++ b/config/initializers/sidekiq.rb @@ -14,8 +14,6 @@ Sidekiq.default_worker_options = { retry: 3 } enable_json_logs = Gitlab.config.sidekiq.log_format == 'json' Sidekiq.configure_server do |config| - require 'rbtrace' if ENV['ENABLE_RBTRACE'] - config.redis = queues_config_hash config.server_middleware do |chain| diff --git a/config/puma.example.development.rb b/config/puma.example.development.rb new file mode 100644 index 00000000000..490c940077a --- /dev/null +++ b/config/puma.example.development.rb @@ -0,0 +1,77 @@ +# frozen_string_literal: true + +# ----------------------------------------------------------------------- +# This file is used by the GDK to generate a default config/puma.rb file +# Note that `/home/git` will be substituted for the actual GDK root +# directory when this file is generated +# ----------------------------------------------------------------------- + +# Load "path" as a rackup file. +# +# The default is "config.ru". +# +rackup 'config.ru' +pidfile '/home/git/gitlab/tmp/pids/puma.pid' +state_path '/home/git/gitlab/tmp/pids/puma.state' + +stdout_redirect '/home/git/gitlab/log/puma.stdout.log', + '/home/git/gitlab/log/puma.stderr.log', + true + +# Configure "min" to be the minimum number of threads to use to answer +# requests and "max" the maximum. +# +# The default is "0, 16". +# +threads 1, 4 + +# By default, workers accept all requests and queue them to pass to handlers. +# When false, workers accept the number of simultaneous requests configured. +# +# Queueing requests generally improves performance, but can cause deadlocks if +# the app is waiting on a request to itself. See https://github.com/puma/puma/issues/612 +# +# When set to false this may require a reverse proxy to handle slow clients and +# queue requests before they reach puma. This is due to disabling HTTP keepalive +queue_requests false + +# Bind the server to "url". "tcp://", "unix://" and "ssl://" are the only +# accepted protocols. +bind 'unix:///home/git/gitlab.socket' + +workers 2 + +require_relative "/home/git/gitlab/lib/gitlab/cluster/lifecycle_events" +require_relative "/home/git/gitlab/lib/gitlab/cluster/puma_worker_killer_initializer" + +on_restart do + # Signal application hooks that we're about to restart + Gitlab::Cluster::LifecycleEvents.do_master_restart +end + +before_fork do + # Signal to the puma killer + Gitlab::Cluster::PumaWorkerKillerInitializer.start @config.options unless ENV['DISABLE_PUMA_WORKER_KILLER'] + + # Signal application hooks that we're about to fork + Gitlab::Cluster::LifecycleEvents.do_before_fork +end + +Gitlab::Cluster::LifecycleEvents.set_puma_options @config.options +on_worker_boot do + # Signal application hooks of worker start + Gitlab::Cluster::LifecycleEvents.do_worker_start +end + +# Preload the application before starting the workers; this conflicts with +# phased restart feature. (off by default) + +preload_app! + +tag 'gitlab-puma-worker' + +# Verifies that all workers have checked in to the master process within +# the given timeout. If not the worker process will be restarted. Default +# value is 60 seconds. +# +worker_timeout 60 diff --git a/config/unicorn.rb.example b/config/unicorn.rb.example index e06cce3e97a..4637eb8bc6e 100644 --- a/config/unicorn.rb.example +++ b/config/unicorn.rb.example @@ -81,22 +81,16 @@ preload_app true # fast LAN. check_client_connection false +require_relative "/home/git/gitlab/lib/gitlab/cluster/lifecycle_events" + before_exec do |server| - # The following is necessary to ensure stale Prometheus metrics don't - # accumulate over time. It needs to be done in this hook as opposed to - # inside an init script to ensure metrics files aren't deleted after new - # unicorn workers start after a SIGUSR2 is received. - if ENV['prometheus_multiproc_dir'] - old_metrics = Dir[File.join(ENV['prometheus_multiproc_dir'], '*.db')] - FileUtils.rm_rf(old_metrics) - end + # Signal application hooks that we're about to restart + Gitlab::Cluster::LifecycleEvents.do_master_restart end before_fork do |server, worker| - # the following is highly recommended for Rails + "preload_app true" - # as there's no need for the master process to hold a connection - defined?(ActiveRecord::Base) && - ActiveRecord::Base.connection.disconnect! + # Signal application hooks that we're about to fork + Gitlab::Cluster::LifecycleEvents.do_before_fork # The following is only recommended for memory/DB-constrained # installations. It is not needed if your system can house @@ -124,25 +118,10 @@ before_fork do |server, worker| end after_fork do |server, worker| - # Unicorn clears out signals before it forks, so rbtrace won't work - # unless it is enabled after the fork. - require 'rbtrace' if ENV['ENABLE_RBTRACE'] + # Signal application hooks of worker start + Gitlab::Cluster::LifecycleEvents.do_worker_start # per-process listener ports for debugging/admin/migrations # addr = "127.0.0.1:#{9293 + worker.nr}" # server.listen(addr, :tries => -1, :delay => 5, :tcp_nopush => true) - - # the following is *required* for Rails + "preload_app true", - defined?(ActiveRecord::Base) && - ActiveRecord::Base.establish_connection - - # reset prometheus client, this will cause any opened metrics files to be closed - defined?(::Prometheus::Client.reinitialize_on_pid_change) && - Prometheus::Client.reinitialize_on_pid_change - - # if preload_app is true, then you may also want to check and - # restart any other shared sockets/descriptors such as Memcached, - # and Redis. TokyoCabinet file handles are safe to reuse - # between any number of forked children (assuming your kernel - # correctly implements pread()/pwrite() system calls) end diff --git a/config/unicorn.rb.example.development b/config/unicorn.rb.example.development index f31df66015a..f7541bb9d55 100644 --- a/config/unicorn.rb.example.development +++ b/config/unicorn.rb.example.development @@ -1,32 +1,61 @@ +# frozen_string_literal: true + +# ------------------------------------------------------------------------- +# This file is used by the GDK to generate a default config/unicorn.rb file +# Note that `/home/git` will be substituted for the actual GDK root +# directory when this file is generated +# ------------------------------------------------------------------------- + worker_processes 2 timeout 60 +listen '/home/git/gitlab.socket' + preload_app true check_client_connection false +require_relative "/home/git/gitlab/lib/gitlab/cluster/lifecycle_events" + +before_exec do |server| + # Signal application hooks that we're about to restart + Gitlab::Cluster::LifecycleEvents.do_master_restart +end + before_fork do |server, worker| - # the following is highly recommended for Rails + "preload_app true" - # as there's no need for the master process to hold a connection - defined?(ActiveRecord::Base) && - ActiveRecord::Base.connection.disconnect! - - if /darwin/ =~ RUBY_PLATFORM - require 'fiddle' - - # Dynamically load Foundation.framework, ~implicitly~ initialising - # the Objective-C runtime before any forking happens in Unicorn - # - # From https://bugs.ruby-lang.org/issues/14009 - Fiddle.dlopen '/System/Library/Frameworks/Foundation.framework/Foundation' + # Signal application hooks that we're about to fork + Gitlab::Cluster::LifecycleEvents.do_before_fork + + # The following is only recommended for memory/DB-constrained + # installations. It is not needed if your system can house + # twice as many worker_processes as you have configured. + # + # This allows a new master process to incrementally + # phase out the old master process with SIGTTOU to avoid a + # thundering herd (especially in the "preload_app false" case) + # when doing a transparent upgrade. The last worker spawned + # will then kill off the old master process with a SIGQUIT. + old_pid = "#{server.config[:pid]}.oldbin" + if old_pid != server.pid + begin + sig = (worker.nr + 1) >= server.worker_processes ? :QUIT : :TTOU + Process.kill(sig, File.read(old_pid).to_i) + rescue Errno::ENOENT, Errno::ESRCH + end end + # + # Throttle the master from forking too quickly by sleeping. Due + # to the implementation of standard Unix signal handlers, this + # helps (but does not completely) prevent identical, repeated signals + # from being lost when the receiving process is busy. + # sleep 1 end after_fork do |server, worker| - # Unicorn clears out signals before it forks, so rbtrace won't work - # unless it is enabled after the fork. - require 'rbtrace' if ENV['ENABLE_RBTRACE'] + # Signal application hooks of worker start + Gitlab::Cluster::LifecycleEvents.do_worker_start - # the following is *required* for Rails + "preload_app true", - defined?(ActiveRecord::Base) && - ActiveRecord::Base.establish_connection + # per-process listener ports for debugging/admin/migrations + # addr = "127.0.0.1:#{9293 + worker.nr}" + # server.listen(addr, :tries => -1, :delay => 5, :tcp_nopush => true) end + -- cgit v1.2.3